You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2015/04/06 20:08:38 UTC

[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2900 - Still Failing

Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2900/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:
shard1 is not consistent.  Got 1023 from http://127.0.0.1:10050/p_/z/collection1lastClient and got 295 from http://127.0.0.1:10059/p_/z/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 1023 from http://127.0.0.1:10050/p_/z/collection1lastClient and got 295 from http://127.0.0.1:10059/p_/z/collection1
	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:52540/fmq/gz/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:52540/fmq/gz/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:52540 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50643/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50643/collection1
	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50643/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:50643 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:11223 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9494 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/init-core-data-001
   [junit4]   2> 295434 T1269 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 295441 T1269 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 295441 T1270 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 295541 T1269 oasc.ZkTestServer.run start zk server on port:50623
   [junit4]   2> 295542 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 295542 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 295545 T1277 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e5605f4 name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 295545 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 295546 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 295546 T1269 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 295548 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 295549 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 295550 T1280 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@842caa6 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 295550 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 295551 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 295551 T1269 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 295553 T1269 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 295554 T1269 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 295555 T1269 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 295556 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 295557 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 295559 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 295560 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 295661 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 295662 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 295663 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 295663 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 295665 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 295665 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 295667 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 295667 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 295668 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 295669 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 295670 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 295670 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 295672 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 295672 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 295673 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 295674 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 295675 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 295675 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 296963 T1269 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1
   [junit4]   2> 296967 T1269 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 296971 T1269 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:50636
   [junit4]   2> 296971 T1269 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=50635, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/tempDir-001/control/data}
   [junit4]   2> 296971 T1269 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 296972 T1269 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/'
   [junit4]   2> 297015 T1269 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/solr.xml
   [junit4]   2> 297067 T1269 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores
   [junit4]   2> 297069 T1269 oasc.CoreContainer.<init> New CoreContainer 2110879613
   [junit4]   2> 297069 T1269 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/]
   [junit4]   2> 297069 T1269 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/lib
   [junit4]   2> 297070 T1269 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/lib).
   [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 297089 T1269 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 297090 T1269 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 297090 T1269 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 297090 T1269 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 297091 T1269 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50623/solr
   [junit4]   2> 297091 T1269 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 297092 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 297092 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 297094 T1292 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45cc06e8 name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 297095 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 297096 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 297097 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 297098 T1295 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@34504d13 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 297098 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 297100 T1269 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 297101 T1269 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 297103 T1269 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 297104 T1269 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 297105 T1269 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 297107 T1269 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 297108 T1269 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 297109 T1269 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 297110 T1269 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50635_
   [junit4]   2> 297110 T1269 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50635_
   [junit4]   2> 297112 T1269 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 297113 T1269 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 297114 T1269 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 297115 T1269 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:50635_
   [junit4]   2> 297115 T1269 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 297116 T1269 oasc.Overseer.start Overseer (id=93607700461191171-127.0.0.1:50635_-n_0000000000) starting
   [junit4]   2> 297118 T1269 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 297122 T1269 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 297122 T1297 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 297122 T1269 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 297122 T1296 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 297126 T1269 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores
   [junit4]   2> 297128 T1269 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, collection=control_collection, coreNodeName=, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/, config=solrconfig.xml, schema=schema.xml, loadOnStartup=true, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1, name=collection1}
   [junit4]   2> 297128 T1269 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/
   [junit4]   2> 297128 T1269 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 297130 T1299 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 297130 T1299 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 297131 T1299 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 297131 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 297132 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50635_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 297132 T1296 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50635_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 297132 T1296 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 297133 T1296 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 297134 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298131 T1299 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 298132 T1299 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 298132 T1299 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 298132 T1299 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 298133 T1299 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 298133 T1299 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/'
   [junit4]   2> 298147 T1299 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 298149 T1299 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 298156 T1299 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 298172 T1299 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 298210 T1299 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 298311 T1299 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 298318 T1299 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 298623 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 298636 T1299 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 298638 T1299 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 298648 T1299 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 298651 T1299 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 298654 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 298656 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 298656 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 298656 T1299 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 298656 T1299 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 298657 T1299 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data/
   [junit4]   2> 298657 T1299 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ec2b6ef
   [junit4]   2> 298658 T1299 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data
   [junit4]   2> 298658 T1299 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index/
   [junit4]   2> 298658 T1299 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 298658 T1299 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index
   [junit4]   2> 298659 T1299 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=52.0615234375, floorSegmentMB=0.486328125, forceMergeDeletesPctAllowed=10.112383767408401, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 298659 T1299 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6041646a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@65279c84),segFN=segments_1,generation=1}
   [junit4]   2> 298659 T1299 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 298666 T1299 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 298685 T1299 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 298687 T1299 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 298689 T1299 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 298690 T1299 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 298694 T1299 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/update/json/docs,/admin/threads,/config,/replication,/admin/file,/admin/logging,/admin/plugins,/update/csv,standard,/admin/ping,/admin/system,/get,/admin/luke,/admin/segments,/admin/mbeans,/admin/properties,/update,/update/json
   [junit4]   2> 298696 T1299 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 298697 T1299 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 298697 T1299 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 298699 T1299 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 298699 T1299 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 298699 T1299 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=39.86328125, floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=25.115751995354497, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 298700 T1299 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6041646a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@65279c84),segFN=segments_1,generation=1}
   [junit4]   2> 298700 T1299 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 298700 T1299 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@445242be[collection1] main
   [junit4]   2> 298701 T1299 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 298701 T1299 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 298701 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 298702 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 298702 T1299 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 298702 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 298703 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 298703 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 298703 T1299 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 298703 T1299 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 298704 T1299 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 298704 T1300 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@445242be[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 298704 T1303 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50635 collection:control_collection shard:shard1
   [junit4]   2> 298705 T1269 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 298706 T1269 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 298706 T1303 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 298710 T1303 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 298710 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 298711 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 298711 T1303 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 298711 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 298712 T1303 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C279 name=collection1 org.apache.solr.core.SolrCore@1745c55d url=http://127.0.0.1:50635/collection1 node=127.0.0.1:50635_ C279_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:50635, node_name=127.0.0.1:50635_, core=collection1}
   [junit4]   2> 298712 T1303 C:control_collection S:shard1 C279 P50635 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50635/collection1/
   [junit4]   2> 298712 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 298712 T1307 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6ea67455 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 298712 T1303 C:control_collection S:shard1 C279 P50635 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 298713 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 298713 T1303 C:control_collection S:shard1 C279 P50635 oasc.SyncStrategy.syncToMe http://127.0.0.1:50635/collection1/ has no replicas
   [junit4]   2> 298713 T1303 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50635/collection1/ shard1
   [junit4]   2> 298714 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 298714 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298714 T1303 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 298714 T1269 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 298717 T1269 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 298718 T1269 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 298719 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 298719 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 298719 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 298720 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 298720 T1310 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c566433 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 298721 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 298721 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 298722 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 298723 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 298723 T1296 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 298724 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298724 T1307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298769 T1303 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:50635/collection1/ and leader is http://127.0.0.1:50635/collection1/
   [junit4]   2> 298770 T1303 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50635
   [junit4]   2> 298770 T1303 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 298770 T1303 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 298770 T1303 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 298771 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 298772 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50635_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 3
   [junit4]   2> 298773 T1296 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50635_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 298773 T1296 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 298774 T1307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298774 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298878 T1307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298878 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 300125 T1269 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1
   [junit4]   2> 300128 T1269 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001
   [junit4]   2> 300128 T1269 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 300132 T1269 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:50644
   [junit4]   2> 300132 T1269 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/tempDir-001/jetty1, hostContext=/, solrconfig=solrconfig.xml, hostPort=50643}
   [junit4]   2> 300133 T1269 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 300133 T1269 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/'
   [junit4]   2> 300171 T1269 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/solr.xml
   [junit4]   2> 300213 T1269 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores
   [junit4]   2> 300213 T1269 oasc.CoreContainer.<init> New CoreContainer 1370575573
   [junit4]   2> 300214 T1269 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/]
   [junit4]   2> 300214 T1269 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/lib
   [junit4]   2> 300215 T1269 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/lib).
   [junit4]   2> 300226 T1269 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 300226 T1269 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 300227 T1269 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 300227 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 300227 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 300227 T1269 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 300228 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 300228 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 300228 T1269 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 300228 T1269 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 300229 T1269 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 300234 T1269 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 300235 T1269 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 300235 T1269 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 300236 T1269 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 300236 T1269 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50623/solr
   [junit4]   2> 300236 T1269 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 300236 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 300237 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 300239 T1322 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15d18950 name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 300240 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 300242 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 300243 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 300245 T1325 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a93df95 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 300245 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 300248 T1269 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 301258 T1269 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50643_
   [junit4]   2> 301259 T1269 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50643_
   [junit4]   2> 301262 T1269 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 301267 T1269 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores
   [junit4]   2> 301269 T1269 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/, name=collection1, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1, config=solrconfig.xml, loadOnStartup=true, transient=false, coreNodeName=, dataDir=data/, schema=schema.xml}
   [junit4]   2> 301270 T1269 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/
   [junit4]   2> 301270 T1269 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 301273 T1326 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 301273 T1326 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 301275 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 301275 T1326 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 301275 T1326 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 301275 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50643",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50643_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 5
   [junit4]   2> 301276 T1326 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 301276 T1296 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50643",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50643_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 301276 T1296 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 301276 T1296 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 301277 T1326 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 301277 T1326 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 301278 T1296 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 301278 T1325 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 301279 T1325 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 302277 T1326 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 302278 T1326 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 302281 T1326 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 302282 T1326 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 302283 T1326 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 302284 T1326 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/'
   [junit4]   2> 302320 T1326 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 302325 T1326 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 302356 T1326 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 302401 T1326 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 302535 T1326 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 302639 T1326 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 302662 T1326 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 303585 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 303613 T1326 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 303616 T1326 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 303637 T1326 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 303645 T1326 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 303652 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 303653 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 303654 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 303654 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 303655 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 303655 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 303655 T1326 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 303656 T1326 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 303656 T1326 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 303657 T1326 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ec2b6ef
   [junit4]   2> 303658 T1326 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data
   [junit4]   2> 303658 T1326 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 303658 T1326 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 303659 T1326 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 303659 T1326 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=52.0615234375, floorSegmentMB=0.486328125, forceMergeDeletesPctAllowed=10.112383767408401, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 303661 T1326 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5be0b2e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d00428d),segFN=segments_1,generation=1}
   [junit4]   2> 303661 T1326 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 303669 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 303669 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 303670 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 303670 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 303670 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 303671 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 303671 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 303671 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 303672 T1326 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 303704 T1326 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303707 T1326 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303710 T1326 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303713 T1326 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303719 T1326 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/update/json/docs,/admin/threads,/config,/replication,/admin/file,/admin/logging,/admin/plugins,/update/csv,standard,/admin/ping,/admin/system,/get,/admin/luke,/admin/segments,/admin/mbeans,/admin/properties,/update,/update/json
   [junit4]   2> 303722 T1326 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 303724 T1326 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 303724 T1326 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 303726 T1326 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 303726 T1326 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 303727 T1326 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=39.86328125, floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=25.115751995354497, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 303728 T1326 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5be0b2e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d00428d),segFN=segments_1,generation=1}
   [junit4]   2> 303728 T1326 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 303729 T1326 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@83d513f[collection1] main
   [junit4]   2> 303729 T1326 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 303730 T1326 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 303730 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 303731 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 303731 T1326 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 303731 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 303732 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 303732 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 303732 T1326 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 303733 T1326 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 303734 T1327 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@83d513f[collection1] main{Exi

[...truncated too long message...]

96/
   [junit4]   2> 1572522 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:11212/
   [junit4]   2> 1572522 T4874 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1572522 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11208/
   [junit4]   2> 1572522 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11223/
   [junit4]   2> 1572523 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11196/
   [junit4]   2> 1572523 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11212/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=FBFBECE5D5AABA29 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es_ES -Dtests.timezone=America/Rankin_Inlet -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   73.2s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:11223 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1572544 T4874 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest FBFBECE5D5AABA29-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {range_facet_l=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), intDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), text=PostingsFormat(name=Memory doPackFST= false), _version_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), timestamp=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), multiDefault=FSTOrd50, a_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), id=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=es_ES, timezone=America/Rankin_Inlet
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=161230816,total=342884352
   [junit4]   2> NOTE: All tests run in this JVM: [TestIntervalFaceting, XmlUpdateRequestHandlerTest, CoreAdminRequestStatusTest, TestReplicaProperties, TestJettySolrRunner, TestRestManager, TestTrie, CollectionReloadTest, HardAutoCommitTest, MinimalSchemaTest, WordBreakSolrSpellCheckerTest, PolyFieldTest, TestHdfsUpdateLog, TestCursorMarkWithoutUniqueKey, QueryEqualityTest, OverseerRolesTest, TestQueryUtils, DirectUpdateHandlerOptimizeTest, TestFreeTextSuggestions, TriLevelCompositeIdRoutingTest, SpellPossibilityIteratorTest, TestAddFieldRealTimeGet, TestDFRSimilarityFactory, DeleteShardTest, PreAnalyzedUpdateProcessorTest, TestSchemaResource, HdfsSyncSliceTest, ZkCLITest, ShowFileRequestHandlerTest, TestQuerySenderListener, RequiredFieldsTest, SoftAutoCommitTest, TestIndexingPerformance, TestCloudPivotFacet, TestManagedSchemaFieldResource, TestSimpleTrackingShardHandler, TestLFUCache, CustomCollectionTest, TestInitQParser, DistributedQueryComponentCustomSortTest, MBeansHandlerTest, TimeZoneUtilsTest, SolrRequestParserTest, SolrTestCaseJ4Test, TestShardHandlerFactory, FileUtilsTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, ZkNodePropsTest, TestCSVResponseWriter, TestManagedStopFilterFactory, CacheHeaderTest, SolrCmdDistributorTest, TestConfig, TestBM25SimilarityFactory, TestManagedResource, MultiThreadedOCPTest, SyncSliceTest, TestCloudManagedSchemaConcurrent, ChaosMonkeyNothingIsSafeTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 73.22s, 1 test, 1 error <<< FAILURES!

[...truncated 698 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 53 minutes 7 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218990361 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2918 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2918/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:
shard1 is not consistent.  Got 963 from http://127.0.0.1:32047/esw/zr/collection1lastClient and got 272 from http://127.0.0.1:32051/esw/zr/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 963 from http://127.0.0.1:32047/esw/zr/collection1lastClient and got 272 from http://127.0.0.1:32051/esw/zr/collection1
	at __randomizedtesting.SeedInfo.seed([CED94C4AA2172441:468D73900CEB49B9]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9725 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryZkTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/init-core-data-001
   [junit4]   2> 648134 T2610 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 648134 T2610 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /esw/zr
   [junit4]   2> 648141 T2610 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 648141 T2611 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 648241 T2610 oasc.ZkTestServer.run start zk server on port:32033
   [junit4]   2> 648242 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 648243 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 648247 T2618 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d31f642 name:ZooKeeperConnection Watcher:127.0.0.1:32033 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 648247 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 648248 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 648248 T2610 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 648251 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 648252 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 648254 T2621 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@50c588 name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 648254 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 648255 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 648255 T2610 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 648257 T2610 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 648259 T2610 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 648260 T2610 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 648261 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 648262 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 648265 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 648266 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 648268 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 648268 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 648270 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 648271 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 648273 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 648273 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 648275 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 648276 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 648277 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 648278 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 648280 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 648281 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 648282 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 648283 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 648285 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 648285 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 648287 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 648288 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 649512 T2610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1
   [junit4]   2> 649515 T2610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 649517 T2610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32039
   [junit4]   2> 649518 T2610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=32039, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores, hostContext=/esw/zr}
   [junit4]   2> 649518 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 649518 T2610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/'
   [junit4]   2> 649548 T2610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/solr.xml
   [junit4]   2> 649583 T2610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores
   [junit4]   2> 649583 T2610 oasc.CoreContainer.<init> New CoreContainer 235084862
   [junit4]   2> 649584 T2610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/]
   [junit4]   2> 649584 T2610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/lib
   [junit4]   2> 649584 T2610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/lib).
   [junit4]   2> 649594 T2610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 649594 T2610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 649594 T2610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 649596 T2610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 649596 T2610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 649596 T2610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 649601 T2610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 649602 T2610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 649602 T2610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 649602 T2610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 649602 T2610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32033/solr
   [junit4]   2> 649603 T2610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 649603 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 649603 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 649605 T2633 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64a4870 name:ZooKeeperConnection Watcher:127.0.0.1:32033 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 649606 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 649606 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 649608 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 649609 T2636 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74862f9b name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 649609 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 649611 T2610 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 649612 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 649613 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 649615 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 649616 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 649617 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 649618 T2610 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 649619 T2610 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 649620 T2610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32039_esw%2Fzr
   [junit4]   2> 649621 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32039_esw%2Fzr
   [junit4]   2> 649622 T2610 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 649623 T2610 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 649624 T2610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 649625 T2610 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:32039_esw%2Fzr
   [junit4]   2> 649625 T2610 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 649626 T2610 oasc.Overseer.start Overseer (id=93624945334288387-127.0.0.1:32039_esw%2Fzr-n_0000000000) starting
   [junit4]   2> 649627 T2610 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 649631 T2610 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 649632 T2638 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 649632 T2637 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 649632 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 649635 T2610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores
   [junit4]   2> 649636 T2610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, collection=control_collection, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1, shard=, config=solrconfig.xml, dataDir=data/, loadOnStartup=true, name=collection1}
   [junit4]   2> 649637 T2610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/
   [junit4]   2> 649637 T2610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 649637 T2640 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 649638 T2640 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 649638 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 649638 T2640 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 649639 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32039_esw%2Fzr"} current state version: 0
   [junit4]   2> 649639 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32039_esw%2Fzr"}
   [junit4]   2> 649639 T2637 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 649640 T2637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 649641 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 650639 T2640 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 650640 T2640 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 650640 T2640 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 650641 T2640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 650641 T2640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 650641 T2640 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/'
   [junit4]   2> 650659 T2640 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 650661 T2640 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 650671 T2640 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 650692 T2640 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 650740 T2640 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 650741 T2640 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 650750 T2640 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 651106 T2640 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 651109 T2640 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 651111 T2640 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 651116 T2640 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 651136 T2640 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 651136 T2640 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 651137 T2640 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data/
   [junit4]   2> 651137 T2640 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@74cee5ed
   [junit4]   2> 651138 T2640 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data
   [junit4]   2> 651138 T2640 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data/index/
   [junit4]   2> 651138 T2640 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 651139 T2640 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data/index
   [junit4]   2> 651139 T2640 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 651140 T2640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@569f34ac lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6e3b34a1),segFN=segments_1,generation=1}
   [junit4]   2> 651141 T2640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 651146 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 651147 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 651147 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 651147 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 651148 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 651148 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 651148 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 651149 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 651149 T2640 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 651171 T2640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 651173 T2640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 651175 T2640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 651178 T2640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 651183 T2640 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/schema,/admin/properties,/update/csv,/admin/file,/admin/mbeans,/config,/admin/luke,/admin/plugins,/admin/ping,/admin/threads,/update/json,/update/json/docs,/update,/get,/admin/system,/replication,standard,/admin/logging
   [junit4]   2> 651185 T2640 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 651187 T2640 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 651187 T2640 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 651189 T2640 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 651189 T2640 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 651190 T2640 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=46, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15818545914049942]
   [junit4]   2> 651190 T2640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@569f34ac lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6e3b34a1),segFN=segments_1,generation=1}
   [junit4]   2> 651191 T2640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 651191 T2640 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6f7491f8[collection1] main
   [junit4]   2> 651191 T2640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 651192 T2640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 651192 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 651193 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 651193 T2640 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 651193 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 651194 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 651194 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 651194 T2640 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 651195 T2640 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 651195 T2641 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6f7491f8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 651196 T2640 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 651197 T2644 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32039/esw/zr collection:control_collection shard:shard1
   [junit4]   2> 651197 T2610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 651197 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 651198 T2644 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 651200 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 651200 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 651202 T2644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 651202 T2647 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a411a6a name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 651202 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 651203 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 651203 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 651203 T2644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 651204 T2644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 651203 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 651204 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C1808 name=collection1 org.apache.solr.core.SolrCore@43fe35d1 url=http://127.0.0.1:32039/esw/zr/collection1 node=127.0.0.1:32039_esw%2Fzr C1808_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:32039/esw/zr, node_name=127.0.0.1:32039_esw%2Fzr}
   [junit4]   2> 651204 T2644 C:control_collection S:shard1 C1808 P32039 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32039/esw/zr/collection1/
   [junit4]   2> 651206 T2644 C:control_collection S:shard1 C1808 P32039 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 651206 T2644 C:control_collection S:shard1 C1808 P32039 oasc.SyncStrategy.syncToMe http://127.0.0.1:32039/esw/zr/collection1/ has no replicas
   [junit4]   2> 651206 T2644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32039/esw/zr/collection1/ shard1
   [junit4]   2> 651207 T2644 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 651207 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651207 T2647 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 651208 T2610 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 651208 T2610 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 651209 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 651209 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 651211 T2650 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@762874b2 name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 651211 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 651212 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 651212 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 651213 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 651215 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 651216 T2637 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 651216 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651216 T2647 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651219 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 651220 T2637 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 651221 T2647 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651221 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651263 T2644 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:32039/esw/zr/collection1/ and leader is http://127.0.0.1:32039/esw/zr/collection1/
   [junit4]   2> 651264 T2644 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32039/esw/zr
   [junit4]   2> 651264 T2644 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 651264 T2644 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 651264 T2644 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 651265 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 651266 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32039_esw%2Fzr"} current state version: 4
   [junit4]   2> 651267 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32039_esw%2Fzr"}
   [junit4]   2> 651370 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651370 T2647 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 652802 T2610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1
   [junit4]   2> 652804 T2610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001
   [junit4]   2> 652805 T2610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 652808 T2610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32047
   [junit4]   2> 652808 T2610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores, hostContext=/esw/zr, hostPort=32047, solrconfig=solrconfig.xml}
   [junit4]   2> 652808 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 652808 T2610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/'
   [junit4]   2> 652838 T2610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/solr.xml
   [junit4]   2> 652872 T2610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores
   [junit4]   2> 652873 T2610 oasc.CoreContainer.<init> New CoreContainer 1177738588
   [junit4]   2> 652873 T2610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/]
   [junit4]   2> 652873 T2610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/lib
   [junit4]   2> 652874 T2610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/lib).
   [junit4]   2> 652883 T2610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 652883 T2610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 652883 T2610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 652885 T2610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 652885 T2610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 652885 T2610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 652889 T2610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 652890 T2610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 652890 T2610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 652891 T2610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 652891 T2610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32033/solr
   [junit4]   2> 652891 T2610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 652891 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 652892 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 652894 T2662 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2dff8c1a name:ZooKeeperConnection Watcher:127.0.0.1:32033 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 652894 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 652895 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 652896 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 652897 T2665 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4028f3af name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 652897 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 652901 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 653904 T2610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32047_esw%2Fzr
   [junit4]   2> 653905 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32047_esw%2Fzr
   [junit4]   2> 653909 T2610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 653911 T2610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores
   [junit4]   2> 653915 T2610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/, dataDir=data/, loadOnStartup=true, coreNodeName=, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1, collection=collection1, shard=, schema=schema.xml, transient=false, name=collection1}
   [junit4]   2> 653916 T2610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/
   [junit4]   2> 653916 T2610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 653918 T2666 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 653918 T2666 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 653919 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 653920 T2666 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 653920 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32047_esw%2Fzr"} current state version: 5
   [junit4]   2> 653921 T2666 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 653922 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32047_esw%2Fzr"}
   [junit4]   2> 653922 T2666 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 653922 T2637 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 653922 T2637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 653923 T2666 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 653923 T2666 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 653923 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 653924 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 653924 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 654923 T2666 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 654924 T2666 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 654924 T2666 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 654925 T2666 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 654925 T2666 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 654926 T2666 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/'
   [junit4]   2> 654944 T2666 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 654947 T2666 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 654958 T2666 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 654982 T2666 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 655050 T2666 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 655052 T2666 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 655062 T2666 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 655423 T2666 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 655426 T2666 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 655428 T2666 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 655432 T2666 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 655451 T2666 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 655452 T2666 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 655452 T2666 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 655453 T2666 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@74cee5ed
   [junit4]   2> 655454 T2666 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data
   [junit4]   2> 655454 T2666 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 655454 T2666 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 655455 T2666 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 655455 T2666 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 655456 T2666 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5c4528fd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4086295e),segFN=segments_1,generation=1}
   [junit4]   2> 655456 T2666 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 655463 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 655464 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 655464 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 655464 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 655465 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 655465 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 655465 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 655466 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 655466 T2666 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 655486 T2666 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 655488 T2666 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 655491 T2666 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 655493 T2666 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 655502 T2666 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/schema,/admin/properties,/update/csv,/admin/file,/admin/mbeans,/config,/admin/luke,/admin/plugins,/admin/ping,/admin/threads,/update/json,/update/json/docs,/update,/get,/admin/system,/replication,standard,/admin/logging
   [junit4]   2> 655504 T2666 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 655506 T2666 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 655506 T2666 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 655508 T2666 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 655508 T2666 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 655509 T2666 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=46, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15818545914049942]
   [junit4]   2> 655510 T2666 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5c4528fd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4086295e),segFN=segments_1,generation=1}
   [junit4]   2> 655510 T2666 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 655511 T2666 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1e20960f[collection1] main
   [junit4]   2> 655511 T2666 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 655512 T2666 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 655512 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 655513 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 655513 T2666 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 655513 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 655513 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 655514 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 655514 T2666 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 655514 T2666 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 655515 T2667 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1e20960f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 655516 T2666 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 655517 T2670 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32047/esw/zr collection:collection1 shard:shard1
   [junit4]   2> 655517 T2610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 655517 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 655517 T2670 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 655521 T2670 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 655522 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 655522 T2670 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 655522 T2670 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1809 name=collection1 org.apache.solr.core.SolrCore@1f1f534 url=http://127.0.0.1:32047/esw/zr/collection1 node=127.0.0.1:32047_esw%2Fzr C1809_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:32047/esw/zr, node_name=127.0.0.1:32047_esw%2Fzr}
   [junit4]   2> 655523 T2670 C:collection1 S:shard1 C1809 P32047 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32047/esw/zr/collection1/
   [junit4]   2> 655523 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 655523 T2670 C:collection1 S:shard1 C1809 P32047 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 655523 T2670 C:collection1 S:shard1 C1809 P32047 oasc.SyncStrategy.syncToMe http://127.0.0.1:32047/esw/zr/collection1/ has no replicas
   [junit4]   2> 655524 T2670 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32047/esw/zr/collection1/ shard1
   [junit4]   2> 655524 T2670 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 655524 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 655525 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655525 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 655527 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 655528 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 655528 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655529 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 655530 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 655577 T2670 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:32047/esw/zr/collection1/ and leader is http://127.0.0.1:32047/esw/zr/collection1/
   [junit4]   2> 655578 T2670 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32047/esw/zr
   [junit4]   2> 655578 T2670 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 655578 T2670 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 655578 T2670 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 655579 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 655580 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32047_esw%2Fzr"} current state version: 5
   [junit4]   2> 655580 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32047_esw%2Fzr"}
   [junit4]   2> 655581 T2670 C:collection1 S:shard1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 655581 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 655582 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655582 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 656725 T2610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1
   [junit4]   2> 656728 T2610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001
   [junit4]   2> 656728 T2610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 656731 T2610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32051
   [junit4]   2> 656731 T2610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/esw/zr, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/tempDir-001/jetty2, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores, hostPort=32051, solrconfig=solrconfig.xml}
   [junit4]   2> 656732 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 656732 T2610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/'
   [junit4]   2> 656762 T2610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/solr.xml
   [junit4]   2> 656797 T2610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores
   [junit4]   2> 656797 T2610 oasc.CoreContainer.<init> New CoreContainer 192842349
   [junit4]   2> 656797 T2610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/]
   [junit4]   2> 656798 T2610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/lib
   [junit4]   2> 656798 T2610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/lib).
   [junit4]   2> 656807 T2610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 656814 T2610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 656815 T2610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 656815 T2610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 656815 T2610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 656815 T2610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32033/solr
   [junit4]   2> 656816 T2610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 656816 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 656817 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 656818 T2682 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60b18fb0 name:ZooKeeperConnection Watcher:127.0.0.1:32033 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 656819 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 656819 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 656821 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 656822 T2685 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21576f30 name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 656822 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 656825 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 657829 T2610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32051_esw%2Fzr
   [junit4]   2> 657830 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32051_esw%2Fzr
   [junit4]   2> 657833 T2610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 657835 T2610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores
   [junit4]   2> 657838 T2610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, dataDir=data/, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/, loadOnStartup=true, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1, name=collection1, transient=false, schema=schema.xml, coreNodeName=}
   [junit4]   2> 657838 T2610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/
   [junit4]   2> 657838 T2610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 657840 T2686 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 657840 T2686 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 657841 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 657842 T2686 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 657842 T2686 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 657842 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32051/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32051_esw%2Fzr"} current state version: 5
   [junit4]   2> 657843 T2686 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 657843 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32051/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32051_esw%2Fzr"}
   [junit4]   2> 657843 T2637 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 657843 T2637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 657843 T2686 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 657844 T2686 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 657844 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 4
   [junit4]   2> 657845 T2685 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 657845 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 657845 T2685 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 657845 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 658844 T2686 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 658845 T2686 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 658845 T2686 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 658846 T2686 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 658846 T2686 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 658846 T2686 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/'
   [junit4]   2> 658865 T2686 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 658867 T2686 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 658877 T2686 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 658899 T2686 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 658948 T2686 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 658949 T2686 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 658959 T2686 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 659306 T2686 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 659309 T2686 C:collection

[...truncated too long message...]

ds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=492,cumulative_deletesById=226,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=56680,transaction_logs_total_number=1}
   [junit4]   2> 819687 T2729 C:collection1 S:shard1 c:collection1 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 819687 T2729 C:collection1 S:shard1 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 819687 T2729 C:collection1 S:shard1 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 819689 T2729 C:collection1 S:shard1 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 819697 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 819697 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data;done=false>>]
   [junit4]   2> 819697 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data
   [junit4]   2> 819698 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data/index;done=false>>]
   [junit4]   2> 819698 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data/index
   [junit4]   2> 820389 T2736 C:collection1 S:shard1 c:collection1 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 820391 T2610 C:collection1 S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/esw/zr,null}
   [junit4]   2> 820594 T2610 C:collection1 S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:32033 32033
   [junit4]   2> 820765 T2611 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:32033 32033
   [junit4]   2> 820767 T2611 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/aliases.json
   [junit4]   2> 		5	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/overseer_elect/election/93624945334288391-127.0.0.1:32047_esw%2Fzr-n_0000000001
   [junit4]   2> 		2	/solr/collections/collection1/leader_elect/shard1/election/93624945334288391-core_node1-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryZkTest -Dtests.method=test -Dtests.seed=CED94C4AA2172441 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ja -Dtests.timezone=Canada/Saskatchewan -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE  173s J3 | RecoveryZkTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: shard1 is not consistent.  Got 963 from http://127.0.0.1:32047/esw/zr/collection1lastClient and got 272 from http://127.0.0.1:32051/esw/zr/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CED94C4AA2172441:468D73900CEB49B9]:0)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 820776 T2610 C:collection1 S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001
   [junit4]   2> 172645 T2609 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=PostingsFormat(name=LuceneVarGapFixedInterval), a_i=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=LuceneVarGapDocFreqInterval), rnd_b=PostingsFormat(name=LuceneVarGapDocFreqInterval), _version_=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{}, sim=DefaultSimilarity, locale=ja, timezone=Canada/Saskatchewan
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=75957512,total=310378496
   [junit4]   2> NOTE: All tests run in this JVM: [DeleteInactiveReplicaTest, TestFuzzyAnalyzedSuggestions, DeleteShardTest, TestRebalanceLeaders, SchemaVersionSpecificBehaviorTest, SuggesterTest, ConnectionManagerTest, DistribJoinFromCollectionTest, TestCloudSchemaless, TestPivotHelperCode, SortByFunctionTest, UUIDFieldTest, TestReqParamsAPI, TestSimpleTrackingShardHandler, TestSearchPerf, TestJoin, TestUpdate, CurrencyFieldOpenExchangeTest, BasicZkTest, DebugComponentTest, TestHashPartitioner, TestWordDelimiterFilterFactory, OutOfBoxZkACLAndCredentialsProvidersTest, UUIDUpdateProcessorFallbackTest, DirectUpdateHandlerOptimizeTest, ZkNodePropsTest, TestJsonRequest, NumericFieldsTest, TestSolrConfigHandlerConcurrent, HdfsSyncSliceTest, ActionThrottleTest, CacheHeaderTest, TestFieldTypeCollectionResource, TestRequestStatusCollectionAPI, TestBinaryField, TestNonDefinedSimilarityFactory, TestCharFilters, PingRequestHandlerTest, InfoHandlerTest, DirectSolrConnectionTest, ScriptEngineTest, ClusterStateUpdateTest, TestMergePolicyConfig, TestQueryTypes, HighlighterTest, DateRangeFieldTest, TestDefaultSearchFieldResource, TestClassNameShortening, TestJettySolrRunner, RecoveryZkTest]
   [junit4] Completed [153/481] on J3 in 173.44s, 1 test, 1 failure <<< FAILURES!

[...truncated 1047 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:484: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:229: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:510: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1434: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:991: There were test failures: 481 suites, 1911 tests, 1 failure, 62 ignored (22 assumptions)

Total time: 57 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 230896502 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2917 - Still Failing

Posted by Steve Rowe <sa...@gmail.com>.
I logged into the lucene slave and ran ant clean-jars as jenkins on each of the non-4.x workspaces containing *-1.7.6.jar, so we shouldn’t see this particular problem again - here’s the list of affected workspaces:

Lucene-Solr-Tests-trunk-Java7  (obsolete)
Lucene-Solr-SmokeRelease-trunk (disabled)
Lucene-Solr-Tests-5.x-Java7
Lucene-Solr-Tests-trunk-Java8  (disabled)
Lucene-Solr-Clover-trunk       (disabled)
Lucene-Solr-NightlyTests-trunk (disabled)

Steve

> On Apr 9, 2015, at 9:53 AM, Apache Jenkins Server <je...@builds.apache.org> wrote:
> 
> Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2917/
> 
> All tests passed
> 
> Build Log:
> [...truncated 28458 lines...]
> check-licenses:
>     [echo] License check under: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr
> [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jcl-over-slf4j-1.7.6.jar
> [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jcl-over-slf4j-1.7.6.jar.sha1
> 
> [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jul-to-slf4j-1.7.6.jar
> [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jul-to-slf4j-1.7.6.jar.sha1
> [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/slf4j-api-1.7.6.jar
> [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/slf4j-api-1.7.6.jar.sha1
> 
> [...truncated 3 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:536: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:97: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:294: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
> If you recently modified ivy-versions.properties or any module's ivy.xml,
> make sure you run "ant clean-jars jar-checksums" before running precommit.
> 
> Total time: 90 minutes 29 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
> Archived 1 artifacts
> Archive block size is 32768
> Received 0 blocks and 464 bytes
> Compression is 0.0%
> Took 0.18 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
> 
> 
> 
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2917 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2917/

All tests passed

Build Log:
[...truncated 28458 lines...]
check-licenses:
     [echo] License check under: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jcl-over-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jcl-over-slf4j-1.7.6.jar.sha1

 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jul-to-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jul-to-slf4j-1.7.6.jar.sha1
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/slf4j-api-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/slf4j-api-1.7.6.jar.sha1

[...truncated 3 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:97: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:294: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
If you recently modified ivy-versions.properties or any module's ivy.xml,
make sure you run "ant clean-jars jar-checksums" before running precommit.

Total time: 90 minutes 29 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 1 artifacts
Archive block size is 32768
Received 0 blocks and 464 bytes
Compression is 0.0%
Took 0.18 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2916 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2916/

1 tests failed.
REGRESSION:  org.apache.solr.schema.TestCloudSchemaless.test

Error Message:
QUERY FAILED: xpath=/response/arr[@name='fields']/lst/str[@name='name'][.='newTestFieldInt448']  request=/schema/fields?wt=xml  response=<?xml version="1.0" encoding="UTF-8"?> <response>  <lst name="responseHeader">   <int name="status">0</int>   <int name="QTime">1873</int> </lst> <arr name="fields">   <lst>     <str name="name">_version_</str>     <str name="type">long</str>     <bool name="stored">true</bool>     <bool name="indexed">true</bool>   </lst>   <lst>     <str name="name">constantField</str>     <str name="type">tdouble</str>   </lst>   <lst>     <str name="name">id</str>     <str name="type">string</str>     <bool name="stored">true</bool>     <bool name="indexed">true</bool>     <bool name="required">true</bool>     <bool name="multiValued">false</bool>     <bool name="uniqueKey">true</bool>   </lst>   <lst>     <str name="name">newTestFieldInt0</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt1</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt10</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt100</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt101</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt102</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt103</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt104</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt105</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt106</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt107</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt108</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt109</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt11</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt110</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt111</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt112</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt113</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt114</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt115</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt116</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt117</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt118</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt119</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt12</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt120</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt121</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt122</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt123</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt124</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt125</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt126</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt127</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt128</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt129</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt13</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt130</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt131</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt132</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt133</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt134</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt135</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt136</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt137</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt138</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt139</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt14</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt140</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt141</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt142</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt143</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt144</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt145</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt146</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt147</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt148</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt149</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt15</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt150</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt151</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt152</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt153</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt154</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt155</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt156</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt157</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt158</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt159</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt16</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt160</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt161</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt162</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt163</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt164</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt165</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt166</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt167</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt168</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt169</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt17</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt170</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt171</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt172</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt173</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt174</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt175</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt176</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt177</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt178</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt179</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt18</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt180</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt181</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt182</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt183</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt184</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt185</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt186</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt187</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt188</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt189</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt19</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt190</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt191</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt192</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt193</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt194</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt195</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt196</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt197</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt198</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt199</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt2</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt20</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt200</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt201</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt202</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt203</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt204</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt205</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt206</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt207</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt208</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt209</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt21</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt210</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt211</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt212</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt213</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt214</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt215</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt216</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt217</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt218</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt219</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt22</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt220</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt221</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt222</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt223</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt224</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt225</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt226</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt227</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt228</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt229</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt23</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt230</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt231</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt232</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt233</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt234</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt235</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt236</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt237</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt238</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt239</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt24</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt240</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt241</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt242</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt243</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt244</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt245</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt246</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt247</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt248</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt249</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt25</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt250</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt251</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt252</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt253</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt254</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt255</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt256</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt257</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt258</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt259</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt26</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt260</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt261</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt262</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt263</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt264</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt265</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt266</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt267</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt268</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt269</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt27</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt270</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt271</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt272</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt273</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt274</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt275</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt276</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt277</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt278</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt279</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt28</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt280</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt281</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt282</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt283</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt284</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt285</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt286</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt287</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt288</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt289</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt29</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt290</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt291</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt292</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt293</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt294</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt295</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt296</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt297</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt298</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt299</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt3</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt30</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt300</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt301</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt302</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt303</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt304</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt305</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt306</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt307</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt308</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt309</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt31</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt310</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt311</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt312</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt313</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt314</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt315</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt316</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt317</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt318</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt319</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt32</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt320</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt321</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt322</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt323</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt324</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt325</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt326</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt327</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt328</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt329</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt33</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt330</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt331</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt332</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt333</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt334</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt335</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt336</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt337</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt338</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt339</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt34</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt340</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt341</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt342</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt343</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt344</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt345</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt346</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt347</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt348</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt349</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt35</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt350</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt351</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt352</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt353</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt354</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt355</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt356</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt357</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt358</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt359</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt36</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt360</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt361</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt362</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt363</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt364</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt365</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt366</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt367</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt368</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt369</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt37</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt370</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt371</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt372</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt373</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt374</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt375</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt376</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt377</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt378</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt379</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt38</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt380</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt381</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt382</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt383</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt384</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt385</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt386</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt387</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt388</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt389</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt39</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt390</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt391</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt392</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt393</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt394</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt395</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt396</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt397</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt398</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt399</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt4</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt40</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt400</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt401</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt402</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt403</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt404</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt405</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt406</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt407</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt408</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt409</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt41</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt410</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt411</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt412</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt413</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt414</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt415</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt416</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt417</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt418</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt419</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt42</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt420</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt421</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt422</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt423</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt424</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt425</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt426</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt427</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt428</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt429</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt43</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt430</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt431</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt432</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt433</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt434</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt435</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt436</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt437</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt438</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt439</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt44</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt440</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt441</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt442</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt443</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt444</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt445</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt446</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt447</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt449</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt45</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt46</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt47</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt48</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt49</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt5</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt50</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt51</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt52</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt53</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt54</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt55</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt56</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt57</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt58</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt59</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt6</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt60</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt61</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt62</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt63</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt64</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt65</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt66</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt67</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt68</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt69</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt7</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt70</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt71</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt72</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt73</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt74</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt75</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt76</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt77</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt78</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt79</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt8</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt80</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt81</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt82</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt83</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt84</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt85</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt86</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt87</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt88</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt89</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt9</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt90</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt91</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt92</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt93</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt94</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt95</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt96</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt97</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt98</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt99</str>     <str name="type">tlong</str>   </lst> </arr> </response> 

Stack Trace:
java.lang.AssertionError: QUERY FAILED: xpath=/response/arr[@name='fields']/lst/str[@name='name'][.='newTestFieldInt448']  request=/schema/fields?wt=xml  response=<?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
  <int name="status">0</int>
  <int name="QTime">1873</int>
</lst>
<arr name="fields">
  <lst>
    <str name="name">_version_</str>
    <str name="type">long</str>
    <bool name="stored">true</bool>
    <bool name="indexed">true</bool>
  </lst>
  <lst>
    <str name="name">constantField</str>
    <str name="type">tdouble</str>
  </lst>
  <lst>
    <str name="name">id</str>
    <str name="type">string</str>
    <bool name="stored">true</bool>
    <bool name="indexed">true</bool>
    <bool name="required">true</bool>
    <bool name="multiValued">false</bool>
    <bool name="uniqueKey">true</bool>
  </lst>
  <lst>
    <str name="name">newTestFieldInt0</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt1</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt10</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt100</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt101</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt102</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt103</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt104</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt105</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt106</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt107</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt108</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt109</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt11</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt110</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt111</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt112</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt113</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt114</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt115</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt116</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt117</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt118</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt119</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt12</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt120</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt121</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt122</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt123</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt124</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt125</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt126</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt127</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt128</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt129</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt13</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt130</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt131</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt132</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt133</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt134</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt135</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt136</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt137</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt138</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt139</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt14</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt140</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt141</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt142</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt143</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt144</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt145</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt146</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt147</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt148</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt149</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt15</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt150</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt151</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt152</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt153</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt154</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt155</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt156</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt157</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt158</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt159</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt16</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt160</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt161</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt162</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt163</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt164</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt165</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt166</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt167</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt168</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt169</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt17</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt170</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt171</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt172</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt173</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt174</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt175</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt176</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt177</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt178</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt179</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt18</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt180</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt181</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt182</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt183</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt184</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt185</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt186</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt187</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt188</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt189</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt19</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt190</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt191</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt192</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt193</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt194</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt195</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt196</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt197</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt198</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt199</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt2</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt20</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt200</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt201</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt202</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt203</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt204</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt205</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt206</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt207</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt208</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt209</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt21</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt210</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt211</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt212</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt213</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt214</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt215</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt216</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt217</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt218</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt219</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt22</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt220</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt221</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt222</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt223</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt224</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt225</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt226</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt227</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt228</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt229</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt23</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt230</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt231</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt232</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt233</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt234</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt235</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt236</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt237</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt238</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt239</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt24</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt240</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt241</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt242</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt243</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt244</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt245</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt246</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt247</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt248</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt249</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt25</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt250</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt251</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt252</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt253</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt254</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt255</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt256</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt257</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt258</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt259</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt26</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt260</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt261</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt262</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt263</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt264</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt265</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt266</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt267</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt268</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt269</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt27</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt270</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt271</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt272</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt273</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt274</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt275</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt276</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt277</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt278</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt279</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt28</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt280</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt281</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt282</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt283</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt284</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt285</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt286</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt287</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt288</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt289</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt29</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt290</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt291</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt292</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt293</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt294</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt295</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt296</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt297</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt298</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt299</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt3</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt30</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt300</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt301</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt302</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt303</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt304</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt305</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt306</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt307</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt308</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt309</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt31</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt310</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt311</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt312</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt313</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt314</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt315</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt316</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt317</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt318</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt319</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt32</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt320</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt321</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt322</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt323</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt324</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt325</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt326</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt327</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt328</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt329</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt33</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt330</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt331</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt332</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt333</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt334</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt335</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt336</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt337</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt338</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt339</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt34</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt340</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt341</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt342</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt343</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt344</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt345</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt346</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt347</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt348</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt349</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt35</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt350</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt351</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt352</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt353</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt354</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt355</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt356</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt357</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt358</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt359</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt36</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt360</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt361</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt362</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt363</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt364</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt365</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt366</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt367</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt368</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt369</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt37</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt370</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt371</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt372</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt373</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt374</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt375</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt376</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt377</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt378</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt379</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt38</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt380</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt381</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt382</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt383</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt384</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt385</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt386</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt387</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt388</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt389</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt39</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt390</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt391</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt392</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt393</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt394</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt395</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt396</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt397</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt398</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt399</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt4</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt40</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt400</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt401</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt402</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt403</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt404</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt405</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt406</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt407</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt408</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt409</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt41</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt410</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt411</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt412</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt413</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt414</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt415</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt416</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt417</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt418</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt419</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt42</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt420</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt421</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt422</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt423</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt424</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt425</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt426</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt427</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt428</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt429</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt43</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt430</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt431</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt432</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt433</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt434</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt435</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt436</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt437</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt438</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt439</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt44</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt440</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt441</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt442</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt443</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt444</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt445</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt446</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt447</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt449</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt45</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt46</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt47</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt48</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt49</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt5</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt50</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt51</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt52</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt53</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt54</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt55</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt56</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt57</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt58</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt59</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt6</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt60</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt61</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt62</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt63</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt64</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt65</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt66</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt67</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt68</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt69</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt7</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt70</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt71</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt72</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt73</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt74</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt75</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt76</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt77</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt78</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt79</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt8</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt80</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt81</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt82</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt83</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt84</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt85</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt86</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt87</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt88</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt89</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt9</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt90</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt91</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt92</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt93</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt94</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt95</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt96</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt97</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt98</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt99</str>
    <str name="type">tlong</str>
  </lst>
</arr>
</response>

	at __randomizedtesting.SeedInfo.seed([C8B1BCB8C8FBCD32:40E583626607A0CA]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.schema.TestCloudSchemaless.test(TestCloudSchemaless.java:159)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakCon

[...truncated too long message...]

4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt68</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt69</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt7</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt70</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt71</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt72</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt73</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt74</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt75</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt76</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt77</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt78</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt79</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt8</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt80</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt81</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt82</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt83</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt84</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt85</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt86</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt87</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt88</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt89</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt9</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt90</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt91</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt92</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt93</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt94</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt95</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt96</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt97</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt98</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt99</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    > </arr>
   [junit4]    > </response>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C8B1BCB8C8FBCD32:40E583626607A0CA]:0)
   [junit4]    > 	at org.apache.solr.schema.TestCloudSchemaless.test(TestCloudSchemaless.java:159)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1371084 T3547 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.schema.TestCloudSchemaless C8B1BCB8C8FBCD32-001
   [junit4]   2> 101046 T3546 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=et, timezone=Pacific/Honolulu
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=142490696,total=348651520
   [junit4]   2> NOTE: All tests run in this JVM: [TestCoreDiscovery, TestBlobHandler, TestLuceneMatchVersion, TestDynamicLoading, TestDistributedSearch, ExternalCollectionsTest, TestDFRSimilarityFactory, DistributedQueryElevationComponentTest, TestSearcherReuse, ConnectionManagerTest, QueryResultKeyTest, DistribCursorPagingTest, TestBlendedInfixSuggestions, OpenCloseCoreStressTest, TestCloudPivotFacet, SolrCmdDistributorTest, DefaultValueUpdateProcessorTest, TestNonDefinedSimilarityFactory, PluginInfoTest, TestArbitraryIndexDir, TestPerFieldSimilarity, PrimitiveFieldTypeTest, TestRequestStatusCollectionAPI, TestTrie, LeaderInitiatedRecoveryOnCommitTest, HdfsChaosMonkeySafeLeaderTest, BasicFunctionalityTest, ResponseLogComponentTest, TestChildDocTransformer, TestStressUserVersions, TestReplicaProperties, TestImplicitCoreProperties, TestSurroundQueryParser, DistributedSuggestComponentTest, TestSolrQueryParser, SolrXmlInZkTest, BadIndexSchemaTest, TestBadConfig, IndexSchemaRuntimeFieldTest, TestJmxIntegration, TestReload, SpellCheckComponentTest, TestConfigSets, SyncSliceTest, TestRealTimeGet, ChangedSchemaMergeTest, CacheHeaderTest, TestCloudSchemaless]
   [junit4] Completed on J2 in 101.36s, 1 test, 1 failure <<< FAILURES!

[...truncated 763 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 481 suites, 1911 tests, 1 failure, 62 ignored (22 assumptions)

Total time: 57 minutes 51 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 206525730 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2915 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2915/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {   "collection1":{     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "core":"collection1",             "state":"active",             "base_url":"http://127.0.0.1:30988/h/kg",             "node_name":"127.0.0.1:30988_h%2Fkg",             "leader":"true"},           "core_node2":{             "core":"collection1",             "state":"recovering",             "base_url":"http://127.0.0.1:45044/h/kg",             "node_name":"127.0.0.1:45044_h%2Fkg"}}}},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "replicationFactor":"1",     "autoCreated":"true",     "router":{"name":"compositeId"}},   "control_collection":{     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node1":{             "core":"collection1",             "state":"active",             "base_url":"http://127.0.0.1:57657/h/kg",             "node_name":"127.0.0.1:57657_h%2Fkg",             "leader":"true"}}}},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "replicationFactor":"1",     "autoCreated":"true",     "router":{"name":"compositeId"}}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
  "collection1":{
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "core":"collection1",
            "state":"active",
            "base_url":"http://127.0.0.1:30988/h/kg",
            "node_name":"127.0.0.1:30988_h%2Fkg",
            "leader":"true"},
          "core_node2":{
            "core":"collection1",
            "state":"recovering",
            "base_url":"http://127.0.0.1:45044/h/kg",
            "node_name":"127.0.0.1:45044_h%2Fkg"}}}},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "replicationFactor":"1",
    "autoCreated":"true",
    "router":{"name":"compositeId"}},
  "control_collection":{
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node1":{
            "core":"collection1",
            "state":"active",
            "base_url":"http://127.0.0.1:57657/h/kg",
            "node_name":"127.0.0.1:57657_h%2Fkg",
            "leader":"true"}}}},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "replicationFactor":"1",
    "autoCreated":"true",
    "router":{"name":"compositeId"}}}
	at __randomizedtesting.SeedInfo.seed([6F561FCF85559A56:E70220152BA9F7AE]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1920)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:102)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9383 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/init-core-data-001
   [junit4]   2> 208180 T322 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /h/kg
   [junit4]   2> 208192 T322 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 208193 T323 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 208292 T322 oasc.ZkTestServer.run start zk server on port:48644
   [junit4]   2> 208293 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 208294 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 208299 T330 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5529d23e name:ZooKeeperConnection Watcher:127.0.0.1:48644 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 208300 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 208301 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 208302 T322 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 208306 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 208308 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 208310 T333 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29f065a8 name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 208310 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 208311 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 208312 T322 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 208316 T322 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 208318 T322 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 208320 T322 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 208325 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 208326 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 208330 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 208331 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 208334 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 208336 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 208339 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 208340 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 208343 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 208344 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 208346 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 208347 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 208350 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 208351 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 208353 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 208354 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 208356 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 208357 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 208360 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 208361 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 208364 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 208364 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 210925 T322 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1
   [junit4]   2> 210931 T322 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 210937 T322 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58650
   [junit4]   2> 210937 T322 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/h/kg, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/tempDir-001/control/data, hostPort=57657}
   [junit4]   2> 210938 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 210939 T322 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/'
   [junit4]   2> 211003 T322 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/solr.xml
   [junit4]   2> 211082 T322 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores
   [junit4]   2> 211082 T322 oasc.CoreContainer.<init> New CoreContainer 847984524
   [junit4]   2> 211083 T322 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/]
   [junit4]   2> 211083 T322 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/lib
   [junit4]   2> 211085 T322 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/lib).
   [junit4]   2> 211101 T322 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 211101 T322 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 211102 T322 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 211102 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 211103 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 211103 T322 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 211104 T322 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 211104 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 211104 T322 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 211105 T322 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 211106 T322 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 211116 T322 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 211118 T322 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 211119 T322 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 211119 T322 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 211119 T322 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48644/solr
   [junit4]   2> 211120 T322 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 211121 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 211122 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 211124 T345 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b33f19f name:ZooKeeperConnection Watcher:127.0.0.1:48644 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 211125 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 211126 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 211129 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 211131 T348 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38433773 name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 211132 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 211134 T322 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 211136 T322 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 211139 T322 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 211141 T322 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 211143 T322 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 211145 T322 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 211147 T322 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 211149 T322 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 211150 T322 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57657_h%2Fkg
   [junit4]   2> 211151 T322 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57657_h%2Fkg
   [junit4]   2> 211153 T322 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 211155 T322 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 211156 T322 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 211158 T322 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:57657_h%2Fkg
   [junit4]   2> 211158 T322 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 211160 T322 oasc.Overseer.start Overseer (id=93621668344889347-127.0.0.1:57657_h%2Fkg-n_0000000000) starting
   [junit4]   2> 211163 T322 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 211169 T322 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 211170 T350 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 211170 T349 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 211170 T322 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 211175 T322 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores
   [junit4]   2> 211183 T322 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1, shard=, coreNodeName=, name=collection1, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/, config=solrconfig.xml, collection=control_collection, transient=false, loadOnStartup=true}
   [junit4]   2> 211183 T322 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/
   [junit4]   2> 211184 T322 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 211185 T352 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 211185 T352 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 211187 T352 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 211187 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 211188 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 211188 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 211189 T349 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 211189 T349 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 211190 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 212187 T352 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 212188 T352 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 212188 T352 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 212189 T352 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 212189 T352 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 212190 T352 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/'
   [junit4]   2> 212205 T352 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 212207 T352 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 212218 T352 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 212246 T352 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 212334 T352 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 212336 T352 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 212352 T352 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 212772 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 212793 T352 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 212796 T352 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 212812 T352 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 212816 T352 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 212819 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 212820 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 212820 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 212820 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 212821 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 212821 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 212821 T352 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 212822 T352 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 212822 T352 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data/
   [junit4]   2> 212823 T352 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@62f98b99
   [junit4]   2> 212823 T352 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data
   [junit4]   2> 212824 T352 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data/index/
   [junit4]   2> 212824 T352 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 212824 T352 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data/index
   [junit4]   2> 212825 T352 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=88.0810546875, floorSegmentMB=1.7578125, forceMergeDeletesPctAllowed=4.679093429411079, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 212826 T352 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@348d830c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3696a3c8),segFN=segments_1,generation=1}
   [junit4]   2> 212826 T352 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 212840 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 212840 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 212841 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 212841 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 212841 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 212842 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 212842 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 212842 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 212843 T352 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 212855 T352 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 212857 T352 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 212858 T352 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 212860 T352 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 212868 T352 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/segments,/get,/update/json/docs,/update/csv,/admin/ping,/admin/file,/schema,/admin/properties,standard,/admin/luke,/config,/admin/mbeans,/update,/admin/logging,/admin/plugins,/update/json,/admin/threads,/replication
   [junit4]   2> 212874 T352 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 212876 T352 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 212876 T352 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 212877 T352 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 212877 T352 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 212878 T352 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=15.447265625, floorSegmentMB=1.6923828125, forceMergeDeletesPctAllowed=1.7328479877713976, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 212879 T352 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@348d830c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3696a3c8),segFN=segments_1,generation=1}
   [junit4]   2> 212879 T352 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 212880 T352 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@41dbc868[collection1] main
   [junit4]   2> 212881 T352 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 212881 T352 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 212881 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 212882 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 212882 T352 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 212882 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 212883 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 212883 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 212883 T352 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 212884 T352 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 212885 T353 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@41dbc868[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 212885 T352 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 212886 T356 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57657/h/kg collection:control_collection shard:shard1
   [junit4]   2> 212886 T322 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 212887 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 212887 T356 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 212889 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 212890 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 212891 T360 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b182ea8 name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 212891 T356 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 212891 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 212892 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 212893 T322 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 212893 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 212893 T356 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 212894 T356 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 212894 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C27 name=collection1 org.apache.solr.core.SolrCore@289cee0a url=http://127.0.0.1:57657/h/kg/collection1 node=127.0.0.1:57657_h%2Fkg C27_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:57657/h/kg, node_name=127.0.0.1:57657_h%2Fkg}
   [junit4]   2> 212894 T356 C:control_collection S:shard1 C27 P57657 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57657/h/kg/collection1/
   [junit4]   2> 212895 T356 C:control_collection S:shard1 C27 P57657 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 212895 T356 C:control_collection S:shard1 C27 P57657 oasc.SyncStrategy.syncToMe http://127.0.0.1:57657/h/kg/collection1/ has no replicas
   [junit4]   2> 212895 T356 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57657/h/kg/collection1/ shard1
   [junit4]   2> 212895 T356 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 212895 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 212895 T322 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 212895 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 212899 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 212900 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 213003 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 213003 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 213051 T356 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:57657/h/kg/collection1/ and leader is http://127.0.0.1:57657/h/kg/collection1/
   [junit4]   2> 213052 T356 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57657/h/kg
   [junit4]   2> 213052 T356 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 213053 T356 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 213053 T356 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 213055 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 213056 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 213056 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 213160 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 213160 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214571 T322 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1
   [junit4]   2> 214574 T322 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001
   [junit4]   2> 214574 T322 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 214577 T322 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33918
   [junit4]   2> 214577 T322 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, hostPort=30988, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/tempDir-001/jetty1, hostContext=/h/kg, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores}
   [junit4]   2> 214578 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 214578 T322 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/'
   [junit4]   2> 214650 T322 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/solr.xml
   [junit4]   2> 214715 T322 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores
   [junit4]   2> 214715 T322 oasc.CoreContainer.<init> New CoreContainer 1852987774
   [junit4]   2> 214715 T322 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/]
   [junit4]   2> 214716 T322 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/lib
   [junit4]   2> 214719 T322 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/lib).
   [junit4]   2> 214733 T322 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 214733 T322 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 214734 T322 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 214734 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 214734 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 214735 T322 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 214735 T322 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 214735 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 214736 T322 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 214736 T322 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 214736 T322 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 214741 T322 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 214748 T322 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 214748 T322 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 214749 T322 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 214749 T322 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48644/solr
   [junit4]   2> 214749 T322 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 214750 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 214750 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 214752 T372 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@670e068d name:ZooKeeperConnection Watcher:127.0.0.1:48644 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 214753 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 214755 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 214758 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 214760 T375 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@549692e9 name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 214760 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 214766 T322 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 215771 T322 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:30988_h%2Fkg
   [junit4]   2> 215772 T322 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:30988_h%2Fkg
   [junit4]   2> 215775 T322 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 215778 T322 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores
   [junit4]   2> 215781 T322 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/, loadOnStartup=true, transient=false, name=collection1, shard=, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1, coreNodeName=, dataDir=data/}
   [junit4]   2> 215781 T322 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/
   [junit4]   2> 215782 T322 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 215783 T376 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 215784 T376 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 215785 T376 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 215785 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 215787 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 215787 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 215788 T349 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 215788 T349 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 215790 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 215790 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 215790 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 216786 T376 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 216787 T376 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 216788 T376 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 216788 T376 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 216789 T376 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 216789 T376 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/'
   [junit4]   2> 216808 T376 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 216810 T376 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 216831 T376 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 216851 T376 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 216892 T376 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 216893 T376 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 216910 T376 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 217450 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 217471 T376 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 217475 T376 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 217498 T376 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 217503 T376 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 217508 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 217509 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 217510 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 217510 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 217511 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 217511 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 217512 T376 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 217512 T376 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 217513 T376 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 217513 T376 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@62f98b99
   [junit4]   2> 217514 T376 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data
   [junit4]   2> 217515 T376 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 217515 T376 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 217516 T376 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 217516 T376 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=88.0810546875, floorSegmentMB=1.7578125, forceMergeDeletesPctAllowed=4.679093429411079, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 217517 T376 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@749ce18a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@679b66d5),segFN=segments_1,generation=1}
   [junit4]   2> 217518 T376 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 217529 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 217529 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 217530 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 217530 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 217531 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 217531 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 217532 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 217532 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 217533 T376 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 217575 T376 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217579 T376 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217582 T376 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217584 T376 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217595 T376 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/segments,/get,/update/json/docs,/update/csv,/admin/ping,/admin/file,/schema,/admin/properties,standard,/admin/luke,/config,/admin/mbeans,/update,/admin/logging,/admin/plugins,/update/json,/admin/threads,/replication
   [junit4]   2> 217598 T376 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 217600 T376 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 217600 T376 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 217602 T376 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 217602 T376 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 217603 T376 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=15.447265625, floorSegmentMB=1.6923828125, forceMergeDeletesPctAllowed=1.7328479877713976, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 217604 T376 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@749ce18a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@679b66d5),segFN=segments_1,generation=1}
   [junit4]   2> 217605 T376 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 217605 T376 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@56c72790[collection1] main
   [junit4]   2> 217606 T376 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 217607 T376 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 217607 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 217608 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 217608 T376 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 217609 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 217610 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 217610 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 217610 T376 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 217611 T376 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 217612 T377 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@56c72790[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 217613 T376 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 217614 T380 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:30988/h/kg collection:collection1 shard:shard1
   [junit4]   2> 217614 T322 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 217615 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 217615 T380 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 217620 T380 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 217622 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217622 T380 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 217623 T380 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 217623 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> ASYNC  NEW_CORE C28 name=collection1 org.apache.solr.core.SolrCore@6f2ccccc url=http://127.0.0.1:30988/h/kg/collection1 node=127.0.0.1:30988_h%2Fkg C28_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:30988/h/kg, node_name=127.0.0.1:30988_h%2Fkg}
   [junit4]   2> 217623 T380 C:collection1 S:shard1 C28 P30988 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:30988/h/kg/collection1/
   [junit4]   2> 217624 T380 C:collection1 S:shard1 C28 P30988 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 217624 T380 C:collection1 S:shard1 C28 P30988 oasc.SyncStrategy.syncToMe http://127.0.0.1:30988/h/kg/collection1/ has no replicas
   [junit4]   2> 217624 T380 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:30988/h/kg/collection1/ shard1
   [junit4]   2> 217624 T380 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 217625 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217625 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217625 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217628 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217629 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 217733 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217733 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217733 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217781 T380 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:30988/h/kg/collection1/ and leader is http://127.0.0.1:30988/h/kg/collection1/
   [junit4]   2> 217782 T380 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:30988/h/kg
   [junit4]   2> 217782 T380 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 217783 T380 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 217783 T380 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 217785 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217786 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 7
   [junit4]   2> 217787 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 217891 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217892 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217891 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 219542 T322 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1
   [junit4]   2> 219544 T322 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001
   [junit4]   2> 219545 T322 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 219548 T322 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:39770
   [junit4]   2> 219548 T322 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores, hostContext=/h/kg, hostPort=45044, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/tempDir-001/jetty2, solrconfig=solrconfig.xml}
   [junit4]   2> 219549 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 219549 T322 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/'
   [junit4]   2> 219589 T322 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/solr.xml
   [junit4]   2> 219631 T322 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores
   [junit4]   2> 219632 T322 oasc.CoreContainer.<init> New CoreContainer 591304576
   [junit4]   2> 219632 T322 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/]
   [junit4]   2> 219632 T322 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/lib
   [junit4]   2> 219633 T322 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/lib).
   [junit4]   2> 219658 T322 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 219659 T322 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 219660 T322 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 219660 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 219660 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 219661 T322 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 219661 T322 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 219662 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 219662 T322 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 219663 T322 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 219663 T322 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 219690 T322 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 219692 T322 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 219692 T322 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 219693 T322 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 219693 T322 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48644/solr
   [junit4]   2> 219694 T322 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 219694 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 219695 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 219697 T393 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fb8b872 name:ZooKeeperConnection Watcher:127.0.0.1:48644 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 219698 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 219699 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 219701 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 219703 T396 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3155d7fc name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 219703 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 219709 T322 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 220714 T322 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45044_h%2Fkg
   [junit4]   2> 220715 T322 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45044_h%2Fkg
   [junit4]   2> 220718 T322 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 220721 T322 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores
   [junit4]   2> 220724 T322 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, loadOnStartup=true, transient=false, dataDir=data/, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1, coreNodeName=, shard=, config=solrconfig.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/}
   [junit4]   2> 220725 T322 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/
   [junit4]   2> 220726 T322 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 220727 T397 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 220728 T397 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 220729 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 220729 T397 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 220730 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45044/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:45044_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 8
   [junit4]   2> 220730 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45044/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:45044_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 220731 T349 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 220731 T349 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 220732 T396 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 220732 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 220732 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 220732 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 221730 T397 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 221731 T397 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 221732 T397 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 221732 T397 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 221732 T397 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 221733 T397 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/'
   [junit4]   2> 221747 T397 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 221749 T397 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 221760 T397 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 221780 T397 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 221848 T397 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 221849 T397 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 221869 T397 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 222316 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 222332 T397 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 222334 T397 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 222351 T397 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 222355 T397 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 222360 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 222361 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 222361 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 222362 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 222362 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 222363 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 222363 T397 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 222363 T397 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 222364 T397 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/sh

[...truncated too long message...]

tate.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 348207 T441 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 348211 T441 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 348240 T441 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 348240 T441 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data/index.20150409122150355 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data/index.20150409122150355;done=false>>]
   [junit4]   2> 348241 T441 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data/index.20150409122150355
   [junit4]   2> 348241 T441 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data;done=false>>]
   [junit4]   2> 348242 T441 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data
   [junit4]   2> 348905 T442 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 348908 T322 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/h/kg,null}
   [junit4]   2> 349022 T322 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:48644 48644
   [junit4]   2> 349180 T323 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:48644 48644
   [junit4]   2> 349181 T323 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 349182 T322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:30988/h/kg, target: http://127.0.0.1:33918/h/kg
   [junit4]   2> 349182 T322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57657/h/kg, target: http://127.0.0.1:58650/h/kg
   [junit4]   2> 349182 T322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:45044/h/kg, target: http://127.0.0.1:39770/h/kg
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=6F561FCF85559A56 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=pt_PT -Dtests.timezone=Asia/Ulaanbaatar -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE  141s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
   [junit4]    >   "collection1":{
   [junit4]    >     "shards":{"shard1":{
   [junit4]    >         "range":"80000000-7fffffff",
   [junit4]    >         "state":"active",
   [junit4]    >         "replicas":{
   [junit4]    >           "core_node1":{
   [junit4]    >             "core":"collection1",
   [junit4]    >             "state":"active",
   [junit4]    >             "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]    >             "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]    >             "leader":"true"},
   [junit4]    >           "core_node2":{
   [junit4]    >             "core":"collection1",
   [junit4]    >             "state":"recovering",
   [junit4]    >             "base_url":"http://127.0.0.1:45044/h/kg",
   [junit4]    >             "node_name":"127.0.0.1:45044_h%2Fkg"}}}},
   [junit4]    >     "maxShardsPerNode":"1",
   [junit4]    >     "autoAddReplicas":"false",
   [junit4]    >     "replicationFactor":"1",
   [junit4]    >     "autoCreated":"true",
   [junit4]    >     "router":{"name":"compositeId"}},
   [junit4]    >   "control_collection":{
   [junit4]    >     "shards":{"shard1":{
   [junit4]    >         "range":"80000000-7fffffff",
   [junit4]    >         "state":"active",
   [junit4]    >         "replicas":{"core_node1":{
   [junit4]    >             "core":"collection1",
   [junit4]    >             "state":"active",
   [junit4]    >             "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]    >             "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]    >             "leader":"true"}}}},
   [junit4]    >     "maxShardsPerNode":"1",
   [junit4]    >     "autoAddReplicas":"false",
   [junit4]    >     "replicationFactor":"1",
   [junit4]    >     "autoCreated":"true",
   [junit4]    >     "router":{"name":"compositeId"}}}
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([6F561FCF85559A56:E70220152BA9F7AE]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1920)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:102)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 349231 T322 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001
   [junit4]   2> 141056 T321 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=PostingsFormat(name=Asserting), range_facet_l=PostingsFormat(name=Asserting), a_t=PostingsFormat(name=Asserting), text=Lucene50(blocksize=128), id=PostingsFormat(name=Direct), multiDefault=PostingsFormat(name=Asserting), intDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), _version_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=pt_PT, timezone=Asia/Ulaanbaatar
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=103305600,total=232783872
   [junit4]   2> NOTE: All tests run in this JVM: [TestAnalyzeInfixSuggestions, TestElisionMultitermQuery, TestCustomSort, LeaderInitiatedRecoveryOnCommitTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 141.37s, 1 test, 1 failure <<< FAILURES!

[...truncated 1372 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1910 tests, 1 failure, 62 ignored (22 assumptions)

Total time: 58 minutes 50 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 211615867 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2914 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2914/

1 tests failed.
REGRESSION:  org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds

Error Message:
soft529 wasn't fast enough

Stack Trace:
java.lang.AssertionError: soft529 wasn't fast enough
	at __randomizedtesting.SeedInfo.seed([BB89623F8C2FAD9B:EA5D9BBF3D5C9D3C]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9433 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001
   [junit4]   2> 30731 T27 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true)
   [junit4]   2> 30733 T27 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 30734 T27 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 30735 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 30736 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 30737 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 30909 T27 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 30986 T27 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 31119 T27 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 31121 T27 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 31215 T27 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 32831 T27 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 32884 T27 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 32891 T27 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 32943 T27 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 32949 T27 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 32954 T27 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 32960 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 32960 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 32961 T27 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 32962 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 32962 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 32991 T27 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 32991 T27 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 32992 T27 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 33037 T27 oasc.CoreContainer.<init> New CoreContainer 1465666313
   [junit4]   2> 33037 T27 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 33038 T27 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib
   [junit4]   2> 33038 T27 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib).
   [junit4]   2> 33057 T27 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 33057 T27 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 33058 T27 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 33058 T27 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 33058 T27 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 33059 T27 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 33059 T27 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 33060 T27 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 33060 T27 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 33061 T27 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 33061 T27 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 33070 T27 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 33072 T27 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 33072 T27 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 33073 T27 oasc.CoreContainer.load Node Name: testNode
   [junit4]   2> 33076 T27 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, name=collection1, collection=collection1, instanceDir=collection1, transient=false, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001, loadOnStartup=true, shard=shard1}
   [junit4]   2> 33077 T28 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 33078 T28 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 33079 T28 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 33080 T28 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 33203 T28 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 33241 T28 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 33322 T28 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 33323 T28 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 33387 T28 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 33971 T28 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 34010 T28 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 34015 T28 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 34056 T28 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 34065 T28 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 34073 T28 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 34075 T28 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 34075 T28 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 34076 T28 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 34077 T28 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 34078 T28 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 34078 T28 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 34079 T28 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 34091 T28 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/
   [junit4]   2> 34101 T28 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 34187 T28 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001
   [junit4]   2> 34188 T28 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index/
   [junit4]   2> 34188 T28 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 34189 T28 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index
   [junit4]   2> 34190 T28 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 34199 T28 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_1,generation=1}
   [junit4]   2> 34200 T28 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 34614 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 34751 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 34761 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 34762 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 34762 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 34763 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 34763 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 34816 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 34816 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 35079 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 35080 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 35081 T28 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 35108 T28 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35110 T28 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35112 T28 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35114 T28 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35373 T28 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 35480 T28 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/admin/threads,standard,/search-facet-def,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/search-facet-invariants,/update/json,/update,dismax,/terms,mltrh,/replication,/admin/system,/schema,spellCheckCompRH_Direct,spellCheckWithWordbreak_Direct,/update/csv,/admin/plugins,/admin/ping,/admin/properties,/admin/segments,/admin/luke,defaults,/admin/logging,tvrh,/admin/mbeans,/config,lazy,/debug/dump,/mlt,spellCheckWithWordbreak,mock
   [junit4]   2> 35489 T28 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 35491 T28 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 35584 T28 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 35585 T28 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 35634 T28 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 35635 T28 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 35635 T28 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=14, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6545133281336737]
   [junit4]   2> 35637 T28 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_1,generation=1}
   [junit4]   2> 35637 T28 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 35638 T28 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@41341e7[collection1] main
   [junit4]   2> 35668 T28 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 35669 T28 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 35669 T28 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 35670 T28 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 35913 T28 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 36315 T28 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 36316 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 36319 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 36319 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 36319 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 36321 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 36323 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 36324 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 36324 T28 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 36325 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 36326 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 36326 T29 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@41341e7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 36346 T27 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 36355 T27 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 36356 T27 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 36357 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 36358 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 36359 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 36537 T27 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 36575 T27 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 36648 T27 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 36649 T27 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 36706 T27 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 37138 T27 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 37161 T27 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 37163 T27 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 37186 T27 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 37192 T27 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 37196 T27 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 37197 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 37198 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 37198 T27 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 37199 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 37200 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 37200 T27 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 37201 T27 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/
   [junit4]   2> 37201 T27 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 37202 T27 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index/
   [junit4]   2> 37206 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 37207 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 37208 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 37208 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 37208 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 37209 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 37209 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 37210 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 37210 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 37211 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 37212 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 37212 T27 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 37233 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37236 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37238 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37240 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37243 T27 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 37245 T27 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/admin/threads,standard,/search-facet-def,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/search-facet-invariants,/update/json,/update,dismax,/terms,mltrh,/replication,/admin/system,/schema,spellCheckCompRH_Direct,spellCheckWithWordbreak_Direct,/update/csv,/admin/plugins,/admin/ping,/admin/properties,/admin/segments,/admin/luke,defaults,/admin/logging,tvrh,/admin/mbeans,/config,lazy,/debug/dump,/mlt,spellCheckWithWordbreak,mock
   [junit4]   2> 37247 T27 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 37250 T27 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 37256 T27 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 37257 T27 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 37258 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3dd2c849[collection1] main
   [junit4]   2> 37259 T27 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 37259 T27 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 37259 T27 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 37260 T27 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 37262 T27 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 37274 T27 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 37276 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 37276 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 37277 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 37277 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 37277 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 37278 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 37278 T27 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=20, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.858064534025872]
   [junit4]   2> 37278 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 37280 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 37281 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 37282 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 37283 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 37284 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 37285 T31 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3dd2c849[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 37287 T27 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_1,generation=1}
   [junit4]   2> 37287 T27 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 37288 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 37288 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@442cca32[collection1] main
   [junit4]   2> 37300 T27 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 37300 T27 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@367c10a8
   [junit4]   2> 37301 T31 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@442cca32[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 37301 T27 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 37302 T27 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C3 name=collection1 org.apache.solr.core.SolrCore@469681e0
   [junit4]   2> 38767 T27 c:collection1 C3 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1497931605250932736)]} 0 1458
   [junit4]   2> 39257 T33 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 39796 T27 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SoftAutoCommitTest -Dtests.method=testSoftAndHardCommitMaxTimeMixedAdds -Dtests.seed=BB89623F8C2FAD9B -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=el_CY -Dtests.timezone=Europe/Luxembourg -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 3.54s J2 | SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: soft529 wasn't fast enough
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BB89623F8C2FAD9B:EA5D9BBF3D5C9D3C]:0)
   [junit4]    > 	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 39893 T27 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 39893 T27 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 39895 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 39896 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 39897 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 39957 T32 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 40080 T27 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 40134 T27 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 40228 T27 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 40229 T27 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 40284 T33 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4e096ef3[collection1] main
   [junit4]   2> 40285 T33 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 40286 T31 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4e096ef3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1)))}
   [junit4]   2> 40287 T32 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_2,generation=2}
   [junit4]   2> 40287 T32 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 40288 T32 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 40289 T32 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 40321 T27 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 40879 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 40915 T27 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 40920 T27 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 40955 T27 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 40963 T27 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 40970 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 40972 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 40973 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 40973 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 40975 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 40975 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 40976 T27 c:collection1 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 40977 T27 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/
   [junit4]   2> 40978 T27 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 40978 T27 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index/
   [junit4]   2> 40983 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 40984 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 40985 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 40986 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 40986 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 40987 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 40988 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 40990 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 40990 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 40992 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 40992 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 40993 T27 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 41030 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 41033 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 41038 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 41041 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 41047 T27 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 41049 T27 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/admin/threads,standard,/search-facet-def,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/search-facet-invariants,/update/json,/update,dismax,/terms,mltrh,/replication,/admin/system,/schema,spellCheckCompRH_Direct,spellCheckWithWordbreak_Direct,/update/csv,/admin/plugins,/admin/ping,/admin/properties,/admin/segments,/admin/luke,defaults,/admin/logging,tvrh,/admin/mbeans,/config,lazy,/debug/dump,/mlt,spellCheckWithWordbreak,mock
   [junit4]   2> 41052 T27 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 41055 T27 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 41056 T27 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 41056 T27 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 41058 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@e611d79[collection1] main
   [junit4]   2> 41059 T27 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 41060 T27 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 41060 T27 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 41061 T27 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 41065 T27 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 41083 T27 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 41085 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 41085 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 41086 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 41086 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 41087 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 41088 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 41088 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 41090 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 41090 T27 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1831758711, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 41091 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 41093 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 41095 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 41095 T27 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_2,generation=2}
   [junit4]   2> 41096 T27 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 41096 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 41097 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 41097 T34 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@e611d79[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1)))}
   [junit4]   2> 41103 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6067a790[collection1] main
   [junit4]   2> 41105 T27 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 41105 T34 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6067a790[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1)))}
   [junit4]   2> 41105 T27 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@469681e0
   [junit4]   2> 41108 T27 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=1,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=143,transaction_logs_total_number=1}
   [junit4]   2> 41109 T27 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C4 name=collection1 org.apache.solr.core.SolrCore@14d97f8d
   [junit4]   2> 41117 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5000 (1497931608497324032)]} 0 6
   [junit4]   2> 41122 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5001 (1497931608506761216)]} 0 2
   [junit4]   2> 41126 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5002 (1497931608512004096)]} 0 2
   [junit4]   2> 41130 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5003 (1497931608516198400)]} 0 1
   [junit4]   2> 41135 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5004 (1497931608520392704)]} 0 1
   [junit4]   2> 41137 T27 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 41152 T27 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 41152 T27 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 41155 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 41156 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 41157 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 41313 T27 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 41363 T27 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 41458 T27 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 41459 T27 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 41498 T27 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 41618 T36 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 41783 T36 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@67d122a8[collection1] main
   [junit4]   2> 41784 T36 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 41784 T34 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@67d122a8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1) Uninverting(_1(5.2.0):c5)))}
   [junit4]   2> 42043 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 42095 T27 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 42100 T27 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 42130 T27 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 42142 T27 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 42148 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 42149 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 42150 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 42151 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 42152 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 42152 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 42153 T27 c:collection1 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 42154 T27 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/
   [junit4]   2> 42155 T27 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 42156 T27 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index/
   [junit4]   2> 42160 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 42161 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 42162 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 42163 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 42163 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 42164 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 42165 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 42165 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 42166 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 42167 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 42168 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 42169 T27 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 42196 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 42199 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 42202 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 42205 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 42210 T27 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 42213 T27 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/admin/threads,standard,/search-facet-def,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/search-facet-invariants,/update/json,/update,dismax,/terms,mltrh,/replication,/admin/system,/schema,spellCheckCompRH_Direct,spellCheckWithWordbreak_Direct,/update/csv,/admin/plugins,/admin/ping,/admin/properties,/admin/segments,/admin/luke,defaults,/admin/logging,tvrh,/admin/mbeans,/config,lazy,/debug/dump,/mlt,spellCheckWithWordbreak,mock
   [junit4]   2> 42216 T27 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 42218 T27 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 42224 T27 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 42225 T27 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 42226 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@22bf06d6[collection1] main
   [junit4]   2> 42227 T27 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 42228 T27 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 42228 T27 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 42228 T27 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 42231 T27 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 42242 T27 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 42243 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 42243 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 42244 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 42245 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 42245 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 42246 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 42246 T27 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_3,generation=3}
   [junit4]   2> 42246 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 42247 T27 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 42248 T27 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7145006735137542]
   [junit4]   2> 42248 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 42250 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 42252 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 42252 T27 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_3,generation=3}
   [junit4]   2> 42253 T27 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 42253 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 42253 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 42254 T27 c:collection1 oasc.SolrCore.getSearcher WARN [collection1] PERFORMANCE WARNING: Overlapping onDeckSearchers=2
   [junit4]   2> 42255 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 42256 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@22bf06d6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1) Uninverting(_1(5.2.0):c5)))}
   [junit4]   2> 42260 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@710b47d2[collection1] main
   [junit4]   2> 42261 T27 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 42261 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@710b47d2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1) Uninverting(_1(5.2.0):c5)))}
   [junit4]   2> 42262 T27 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@14d97f8d
   [junit4]   2> 42263 T27 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommit maxTime=1200ms,autocommits=0,soft autocommit maxTime=500ms,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=5,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=5,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=143,transaction_logs_total_number=1}
   [junit4]   2> 42264 T27 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C5 name=collection1 org.apache.solr.core.SolrCore@16552add
   [junit4]   2> 42268 T27 c:collection1 C5 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1497931609708429312)]} 0 3
   [junit4]   2> 42271 T27 c:collection1 C5 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 42304 T27 c:collection1 C5 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_4,generation=4}
   [junit4]   2> 42304 T27 c:collection1 C5 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 42306 T27 c:collection1 C5 oass.SolrIndexSearcher.<init> Opening Searcher@28fbf6a1[collection1] main
   [junit4]   2> 42306 T27 c:collection1 C5 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 42307 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@28fbf6a1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(5.2.0):c5) Uninverting(_2(5.2.0):c1)))}
   [junit4]   2> 42309 T27 c:collection1 C5 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 38
   [junit4]   2> 42315 T27 c:collection1 C5 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {delete=[529 (-1497931609756663808)]} 0 4
   [junit4]   2> 42816 T39 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 42826 T39 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7a3335c5[collection1] main
   [junit4]   2> 42827 T39 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 42827 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a3335c5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(5.2.0):c5)))}
   [junit4]   2> 42830 T27 c:collection1 C5 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[550 (1497931610297729024)]} 0 3
   [junit4]   2> 43331 T39 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 43349 T39 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@536fe809[collection1] main
   [junit4]   2> 43350 T39 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 43350 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@536fe809[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(5.2.0):c5) Uninverting(_3(5.2.0):c1)))}
   [junit4]   2> 43516 T38 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 43518 T38 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_4,generation=4}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_5,generation=5}
   [junit4]   2> 43518 T38 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 5
   [junit4]   2> 43520 T38 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 43520 T38 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 45521 T27 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 45523 T27 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 45524 T27 c:collection1 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1465666313
   [junit4]   2> 45525 T27 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@16552add
   [junit4]   2> 45526 T27 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=4,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=2,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=2,cumulative_deletesById=1,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=707,transaction_logs_total_number=3}
   [junit4]   2> 45527 T27 c:collection1 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 45528 T27 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 45529 T27 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 45530 T27 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 45533 T27 c:collection1 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 45533 T27 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001;done=false>>]
   [junit4]   2> 45534 T27 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001
   [junit4]   2> 45535 T27 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index;done=false>>]
   [junit4]   2> 45535 T27 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=el_CY, timezone=Europe/Luxembourg
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=210408840,total=255852544
   [junit4]   2> NOTE: All tests run in this JVM: [TestManagedSchemaFieldTypeResource, SoftAutoCommitTest]
   [junit4] Completed on J2 in 15.17s, 3 tests, 1 failure <<< FAILURES!

[...truncated 1512 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1910 tests, 1 failure, 62 ignored (22 assumptions)

Total time: 60 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 221032214 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2913 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2913/

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
ERROR: SolrIndexSearcher opens=8 closes=7

Stack Trace:
java.lang.AssertionError: ERROR: SolrIndexSearcher opens=8 closes=7
	at __randomizedtesting.SeedInfo.seed([4868EF3346E25AEB]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:496)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:232)
	at sun.reflect.GeneratedMethodAccessor35.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:799)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=3276, name=searcherExecutor-2038-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=3262, name=qtp1606563175-3262, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:503)         at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:389)         at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:103)         at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:229)         at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:89)         at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:781)         at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1643)         at org.apache.solr.update.processor.LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:183)         at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:83)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:1988)         at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:828)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:446)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)         at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)         at org.eclipse.jetty.server.Server.handle(Server.java:368)         at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)         at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)         at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)         at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)         at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:953)         at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)         at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)         at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=3276, name=searcherExecutor-2038-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=3262, name=qtp1606563175-3262, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:503)
        at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:389)
        at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:103)
        at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:229)
        at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:89)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:781)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1643)
        at org.apache.solr.update.processor.LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:183)
        at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:83)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:1988)
        at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:828)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:446)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
        at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
        at org.eclipse.jetty.server.Server.handle(Server.java:368)
        at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
        at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
        at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
        at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
        at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:953)
        at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
        at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
        at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([4868EF3346E25AEB]:0)




Build Log:
[...truncated 10048 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/init-core-data-001
   [junit4]   2> 1172756 T3222 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1172765 T3222 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1172765 T3223 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1172865 T3222 oasc.ZkTestServer.run start zk server on port:33268
   [junit4]   2> 1172866 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1172867 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1172873 T3230 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a882b79 name:ZooKeeperConnection Watcher:127.0.0.1:33268 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1172873 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1172874 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1172874 T3222 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1172878 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1172879 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1172881 T3233 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5e5fc8b5 name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1172881 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1172882 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1172882 T3222 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1172884 T3222 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1172886 T3222 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1172887 T3222 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1172888 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1172889 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1172891 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1172892 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1172995 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1172996 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1172998 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1172998 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1173000 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1173000 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1173002 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1173002 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1173004 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1173005 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1173006 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1173007 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1173008 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1173009 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1173010 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1173011 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1173012 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1173013 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1174614 T3222 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1
   [junit4]   2> 1174621 T3222 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1174628 T3222 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33280
   [junit4]   2> 1174628 T3222 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=33280, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/tempDir-001/control/data}
   [junit4]   2> 1174629 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 1174630 T3222 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/'
   [junit4]   2> 1174700 T3222 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/solr.xml
   [junit4]   2> 1174835 T3222 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores
   [junit4]   2> 1174836 T3222 oasc.CoreContainer.<init> New CoreContainer 930892105
   [junit4]   2> 1174837 T3222 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/]
   [junit4]   2> 1174837 T3222 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/lib
   [junit4]   2> 1174839 T3222 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/lib).
   [junit4]   2> 1174856 T3222 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1174857 T3222 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1174857 T3222 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1174858 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1174858 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1174859 T3222 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1174859 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1174860 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1174860 T3222 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1174861 T3222 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1174861 T3222 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1174879 T3222 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1174881 T3222 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1174882 T3222 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1174882 T3222 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1174883 T3222 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33268/solr
   [junit4]   2> 1174883 T3222 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1174884 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1174885 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1174888 T3245 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1fd45422 name:ZooKeeperConnection Watcher:127.0.0.1:33268 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1174888 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1174889 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1174894 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1174897 T3248 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@65435e59 name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1174897 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1174901 T3222 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1174904 T3222 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1174907 T3222 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1174910 T3222 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1174915 T3222 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1174917 T3222 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1174919 T3222 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1174921 T3222 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1174923 T3222 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33280_
   [junit4]   2> 1174925 T3222 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33280_
   [junit4]   2> 1174927 T3222 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1174930 T3222 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1174932 T3222 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1174934 T3222 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:33280_
   [junit4]   2> 1174935 T3222 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1174937 T3222 oasc.Overseer.start Overseer (id=93619858301714435-127.0.0.1:33280_-n_0000000000) starting
   [junit4]   2> 1174939 T3222 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1174948 T3222 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1174950 T3250 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1174950 T3222 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1174950 T3249 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1174957 T3222 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores
   [junit4]   2> 1174967 T3222 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, config=solrconfig.xml, transient=false, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1, name=collection1, shard=, loadOnStartup=true, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/, dataDir=data/}
   [junit4]   2> 1174968 T3222 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/
   [junit4]   2> 1174969 T3222 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1174970 T3252 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1174970 T3252 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1174972 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1174972 T3252 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1174973 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33280_",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 1174974 T3249 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33280_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1174974 T3249 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1174974 T3249 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1174976 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1175972 T3252 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1175973 T3252 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1175974 T3252 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1175974 T3252 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1175976 T3252 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1175977 T3252 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/'
   [junit4]   2> 1176004 T3252 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1176008 T3252 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1176031 T3252 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1176086 T3252 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1176176 T3252 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1176279 T3252 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1176291 T3252 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1176758 T3252 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1176761 T3252 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1176763 T3252 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1176768 T3252 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1176796 T3252 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1176797 T3252 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 1176801 T3252 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/
   [junit4]   2> 1176802 T3252 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a7655aa
   [junit4]   2> 1176806 T3252 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data
   [junit4]   2> 1176810 T3252 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1176813 T3252 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1176815 T3252 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index
   [junit4]   2> 1176816 T3252 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=32.4326171875, floorSegmentMB=0.822265625, forceMergeDeletesPctAllowed=24.40693562206468, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6199579108735648
   [junit4]   2> 1176826 T3252 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1176826 T3252 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1176836 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1176837 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1176837 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1176837 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1176838 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1176838 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1176839 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1176839 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1176840 T3252 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1176903 T3252 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1176908 T3252 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1176914 T3252 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1176917 T3252 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1176924 T3252 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/admin/threads,standard,/admin/logging,/admin/luke,/update/json,/update/json/docs,/admin/ping,/admin/file,/admin/mbeans,/admin/system,/update/csv,/admin/plugins,/replication,/get,/admin/properties,/update,/admin/segments,/schema
   [junit4]   2> 1176929 T3252 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1176930 T3252 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1176930 T3252 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1176934 T3252 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1176935 T3252 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1176937 T3252 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=72.5986328125, floorSegmentMB=1.576171875, forceMergeDeletesPctAllowed=29.48913303940835, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.892008954113685
   [junit4]   2> 1176939 T3252 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1176940 T3252 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1176941 T3252 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6cec9e83[collection1] main
   [junit4]   2> 1176941 T3252 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1176943 T3252 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1176943 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1176944 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1176944 T3252 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1176945 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1176945 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1176946 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1176946 T3252 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1176947 T3252 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1176948 T3253 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6cec9e83[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1176949 T3252 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1176952 T3256 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:33280 collection:control_collection shard:shard1
   [junit4]   2> 1176954 T3222 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 1176954 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1176955 T3256 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1176958 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1176959 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1176962 T3259 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40d62081 name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1176962 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1176963 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1176964 T3222 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1176970 T3222 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1176971 T3256 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1176974 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1176974 T3256 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1176974 T3256 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C532 name=collection1 org.apache.solr.core.SolrCore@66c2b8de url=http://127.0.0.1:33280/collection1 node=127.0.0.1:33280_ C532_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:33280, node_name=127.0.0.1:33280_}
   [junit4]   2> 1176975 T3256 C:control_collection S:shard1 C532 P33280 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33280/collection1/
   [junit4]   2> 1176975 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1176975 T3256 C:control_collection S:shard1 C532 P33280 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1176976 T3256 C:control_collection S:shard1 C532 P33280 oasc.SyncStrategy.syncToMe http://127.0.0.1:33280/collection1/ has no replicas
   [junit4]   2> 1176976 T3256 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33280/collection1/ shard1
   [junit4]   2> 1176976 T3256 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1176978 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1176978 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1176984 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1176986 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1177091 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1177092 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1177139 T3256 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:33280/collection1/ and leader is http://127.0.0.1:33280/collection1/
   [junit4]   2> 1177140 T3256 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:33280
   [junit4]   2> 1177140 T3256 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1177140 T3256 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1177140 T3256 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1177143 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1177145 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33280_",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 1177146 T3249 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33280_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1177251 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1177251 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1180447 T3222 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1
   [junit4]   2> 1180453 T3222 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001
   [junit4]   2> 1180454 T3222 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1180461 T3222 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33284
   [junit4]   2> 1180461 T3222 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostContext=/, hostPort=33284}
   [junit4]   2> 1180462 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 1180462 T3222 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/'
   [junit4]   2> 1180551 T3222 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/solr.xml
   [junit4]   2> 1180683 T3222 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores
   [junit4]   2> 1180683 T3222 oasc.CoreContainer.<init> New CoreContainer 863431378
   [junit4]   2> 1180684 T3222 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/]
   [junit4]   2> 1180684 T3222 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/lib
   [junit4]   2> 1180687 T3222 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/lib).
   [junit4]   2> 1180702 T3222 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1180702 T3222 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1180703 T3222 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1180703 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1180703 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1180704 T3222 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1180704 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1180705 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1180705 T3222 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1180705 T3222 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1180706 T3222 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1180725 T3222 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1180727 T3222 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1180727 T3222 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1180727 T3222 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1180728 T3222 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33268/solr
   [junit4]   2> 1180728 T3222 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1180729 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1180730 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1180733 T3271 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@254dca1c name:ZooKeeperConnection Watcher:127.0.0.1:33268 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1180733 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1180734 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1180739 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1180741 T3274 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a2ea17c name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1180742 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1180751 T3222 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1181759 T3222 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33284_
   [junit4]   2> 1181760 T3222 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33284_
   [junit4]   2> 1181764 T3222 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1181766 T3222 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores
   [junit4]   2> 1181768 T3222 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/, schema=schema.xml, coreNodeName=, transient=false, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1, config=solrconfig.xml, dataDir=data/, name=collection1, loadOnStartup=true}
   [junit4]   2> 1181769 T3222 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/
   [junit4]   2> 1181769 T3222 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1181770 T3275 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1181770 T3275 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1181771 T3275 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1181771 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1181772 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33284_",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 1181772 T3249 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33284_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1181773 T3249 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1181773 T3249 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1181774 T3274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1181774 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1181774 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1182771 T3275 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1182772 T3275 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1182773 T3275 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1182773 T3275 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1182775 T3275 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1182776 T3275 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1182802 T3275 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1182807 T3275 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1182825 T3275 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1182868 T3275 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1182952 T3275 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1183054 T3275 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1183074 T3275 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1183668 T3275 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1183674 T3275 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1183677 T3275 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1183686 T3275 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1183718 T3275 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1183718 T3275 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 1183721 T3275 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1183721 T3275 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a7655aa
   [junit4]   2> 1183724 T3275 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1183725 T3275 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1183726 T3275 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1183728 T3275 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1183728 T3275 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=32.4326171875, floorSegmentMB=0.822265625, forceMergeDeletesPctAllowed=24.40693562206468, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6199579108735648
   [junit4]   2> 1183735 T3275 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1183736 T3275 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1183748 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1183748 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1183749 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1183749 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1183750 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1183750 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1183751 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1183751 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1183752 T3275 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1183811 T3275 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1183817 T3275 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1183822 T3275 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1183828 T3275 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1183844 T3275 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/admin/threads,standard,/admin/logging,/admin/luke,/update/json,/update/json/docs,/admin/ping,/admin/file,/admin/mbeans,/admin/system,/update/csv,/admin/plugins,/replication,/get,/admin/properties,/update,/admin/segments,/schema
   [junit4]   2> 1183849 T3275 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1183850 T3275 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1183850 T3275 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1183854 T3275 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1183855 T3275 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1183858 T3275 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=72.5986328125, floorSegmentMB=1.576171875, forceMergeDeletesPctAllowed=29.48913303940835, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.892008954113685
   [junit4]   2> 1183860 T3275 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1183861 T3275 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1183862 T3275 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7b2b64a1[collection1] main
   [junit4]   2> 1183862 T3275 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1183863 T3275 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1183864 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1183864 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1183865 T3275 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1183865 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1183866 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1183866 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1183867 T3275 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1183867 T3275 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1183869 T3276 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7b2b64a1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1183869 T3275 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1183871 T3279 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:33284 collection:collection1 shard:shard1
   [junit4]   2> 1183872 T3222 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 1183872 T3279 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1183872 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1183879 T3279 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1183883 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1183884 T3279 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1183884 T3279 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1183884 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> ASYNC  NEW_CORE C533 name=collection1 org.apache.solr.core.SolrCore@6d5604b6 url=http://127.0.0.1:33284/collection1 node=127.0.0.1:33284_ C533_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:33284, node_name=127.0.0.1:33284_}
   [junit4]   2> 1183884 T3279 C:collection1 S:shard1 C533 P33284 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33284/collection1/
   [junit4]   2> 1183885 T3279 C:collection1 S:shard1 C533 P33284 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1183885 T3279 C:collection1 S:shard1 C533 P33284 oasc.SyncStrategy.syncToMe http://127.0.0.1:33284/collection1/ has no replicas
   [junit4]   2> 1183885 T3279 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33284/collection1/ shard1
   [junit4]   2> 1183886 T3279 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1183886 T3274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183886 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183886 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183891 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1183894 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 1183999 T3274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183999 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183999 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1184044 T3279 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:33284/collection1/ and leader is http://127.0.0.1:33284/collection1/
   [junit4]   2> 1184045 T3279 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:33284
   [junit4]   2> 1184045 T3279 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1184045 T3279 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1184046 T3279 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1184047 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1184048 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33284_",
   [junit4]   2> 	  "roles":null} current state version: 7
   [junit4]   2> 1184049 T3249 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33284_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1184154 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1184154 T3274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1184154 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1187313 T3222 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores/collection1
   [junit4]   2> 1187318 T3222 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001
   [junit4]   2> 1187319 T3222 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1187324 T3222 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33289
   [junit4]   2> 1187325 T3222 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/tempDir-001/jetty2, hostPort=33289, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores}
   [junit4]   2> 1187326 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 1187327 T3222 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/'
   [junit4]   2> 1187444 T3222 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/solr.xml
   [junit4]   2> 1187581 T3222 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores
   [junit4]   2> 1187582 T3222 oasc.CoreContainer.<init> New CoreContainer 83748162
   [junit4]   2> 1187582 T3222 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/]
   [junit4]   2> 1187583 T3222 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/lib
   [junit4]   2> 1187586 T3222 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/lib).
   [junit4]   2> 1187605 T3222 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1187605 T3222 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1187606 T3222 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1187606 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1187606 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1187607 T3222 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1187607 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1187607 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1187608 T3222 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1187608 T3222 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1187608 T3222 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1187626 T3222 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1187629 T3222 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1187629 T3222 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1187630 T3222 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1187630 T3222 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33268/solr
   [junit4]   2> 1187631 T3222 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1187631 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1187632 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1187636 T3291 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a348bc2 name:ZooKeeperConnection Watcher:127.0.0.1:33268 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1187636 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1187637 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1187642 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1187644 T3294 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16ee53d9 name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1187644 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1187653 T3222 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1188661 T3222 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33289_
   [junit4]   2> 1188663 T3222 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33289_
   [junit4]   2> 1188669 T3222 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1188674 T3222 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores
   [junit4]   2> 1188685 T3222 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores/collection1, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores/collection1/, schema=schema.xml, transient=false, config=solrconfig.xml, loadOnStartup=true, coreNodeName=, shard=, dataDir=data/}
   [junit4]   2> 1188686 T3222 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/sol

[...truncated too long message...]

rmat(compressionMode=DUMMY, chunkSize=8, maxDocsPerChunk=79, blockSize=578), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=8, blockSize=578)), sim=DefaultSimilarity, locale=sv, timezone=Mexico/BajaSur
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=123053320,total=283115520
   [junit4]   2> NOTE: All tests run in this JVM: [DistribDocExpirationUpdateProcessorTest, StatsComponentTest, TestReload, ExitableDirectoryReaderTest, TestSweetSpotSimilarityFactory, ParsingFieldUpdateProcessorsTest, TimeZoneUtilsTest, TestFieldCollectionResource, CollectionReloadTest, TestSerializedLuceneMatchVersion, TestStressUserVersions, SimpleFacetsTest, HighlighterMaxOffsetTest, TestLRUStatsCache, TestNRTOpen, TestJsonRequest, TestSolr4Spatial2, ZkNodePropsTest, SortByFunctionTest, TestJsonFacets, MoreLikeThisHandlerTest, HdfsUnloadDistributedZkTest, SuggesterFSTTest, TestJmxIntegration, DebugComponentTest, CSVRequestHandlerTest, TestStandardQParsers, TestJettySolrRunner, TestAddFieldRealTimeGet, HdfsChaosMonkeySafeLeaderTest, TestRandomMergePolicy, TermVectorComponentDistributedTest, DistributedQueryComponentCustomSortTest, TestManagedSynonymFilterFactory, TestZkChroot, SOLR749Test, BlockDirectoryTest, TestArbitraryIndexDir, SSLMigrationTest, CursorMarkTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, DeleteReplicaTest, TestSearchPerf, CollectionsAPIDistributedZkTest, TestSolrConfigHandlerConcurrent, DistribCursorPagingTest, RequiredFieldsTest, UUIDFieldTest, XmlUpdateRequestHandlerTest, RegexBoostProcessorTest, URLClassifyProcessorTest, AlternateDirectoryTest, TestCloudInspectUtil, TestSchemaManager, JSONWriterTest, TestManagedSchema, DistribJoinFromCollectionTest, BasicDistributedZk2Test, TestDistributedGrouping, ChaosMonkeyNothingIsSafeTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeyNothingIsSafeTest -Dtests.seed=4868EF3346E25AEB -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sv -Dtests.timezone=Mexico/BajaSur -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J3 | ChaosMonkeyNothingIsSafeTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ERROR: SolrIndexSearcher opens=8 closes=7
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4868EF3346E25AEB]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:496)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:232)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   [junit4]    >    1) Thread[id=3276, name=searcherExecutor-2038-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    2) Thread[id=3262, name=qtp1606563175-3262, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Object.wait(Object.java:503)
   [junit4]    >         at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:389)
   [junit4]    >         at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:103)
   [junit4]    >         at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:229)
   [junit4]    >         at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:89)
   [junit4]    >         at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:781)
   [junit4]    >         at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1643)
   [junit4]    >         at org.apache.solr.update.processor.LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:183)
   [junit4]    >         at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:83)
   [junit4]    >         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
   [junit4]    >         at org.apache.solr.core.SolrCore.execute(SolrCore.java:1988)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:828)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:446)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]    >         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
   [junit4]    >         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
   [junit4]    >         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
   [junit4]    >         at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
   [junit4]    >         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
   [junit4]    >         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
   [junit4]    >         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
   [junit4]    >         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
   [junit4]    >         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
   [junit4]    >         at org.eclipse.jetty.server.Server.handle(Server.java:368)
   [junit4]    >         at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
   [junit4]    >         at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
   [junit4]    >         at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
   [junit4]    >         at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
   [junit4]    >         at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:953)
   [junit4]    >         at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
   [junit4]    >         at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
   [junit4]    >         at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4868EF3346E25AEB]:0)
   [junit4] Completed on J3 in 487.36s, 1 test, 1 failure, 1 error <<< FAILURES!

[...truncated 701 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1910 tests, 2 suite-level errors, 62 ignored (22 assumptions)

Total time: 59 minutes 45 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 210258778 bytes
Compression is 0.0%
Took 38 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2912 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2912/

All tests passed

Build Log:
[...truncated 28423 lines...]
check-licenses:
     [echo] License check under: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jcl-over-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jcl-over-slf4j-1.7.6.jar.sha1

 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jul-to-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jul-to-slf4j-1.7.6.jar.sha1
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/slf4j-api-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/slf4j-api-1.7.6.jar.sha1

[...truncated 3 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:90: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:256: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
If you recently modified ivy-versions.properties or any module's ivy.xml,
make sure you run "ant clean-jars jar-checksums" before running precommit.

Total time: 87 minutes 37 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 1 artifacts
Archive block size is 32768
Received 0 blocks and 464 bytes
Compression is 0.0%
Took 12 ms
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2911 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2911/

1 tests failed.
FAILED:  org.apache.solr.cloud.TestRebalanceLeaders.test

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:20802/zg, http://127.0.0.1:20781/zg, http://127.0.0.1:20767/zg, http://127.0.0.1:20792/zg, http://127.0.0.1:20743/zg]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:20802/zg, http://127.0.0.1:20781/zg, http://127.0.0.1:20767/zg, http://127.0.0.1:20792/zg, http://127.0.0.1:20743/zg]
	at __randomizedtesting.SeedInfo.seed([ED4E1CF455B33B43:651A232EFB4F56BB]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:349)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1067)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.TestRebalanceLeaders.issueCommands(TestRebalanceLeaders.java:281)
	at org.apache.solr.cloud.TestRebalanceLeaders.rebalanceLeaderTest(TestRebalanceLeaders.java:108)
	at org.apache.solr.cloud.TestRebalanceLeaders.test(TestRebalanceLeaders.java:74)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:20767/zg: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 {trace=java.lang.AssertionError
	at org.apache.solr.common.util.ByteUtils.UTF8toUTF16(ByteUtils.java:36)
	at org.apache.solr.common.util.ByteUtils.UTF8toUTF16(ByteUtils.java:64)
	at org.apache.solr.common.cloud.ZkStateReader.fromJSON(ZkStateReader.java:137)
	at org.apache.solr.common.cloud.ZkNodeProps.load(ZkNodeProps.java:92)
	at org.apache.solr.cloud.DistributedQueue.containsTaskWithRequestId(DistributedQueue.java:127)
	at org.apache.solr.handler.admin.CollectionsHandler.overseerCollectionQueueContains(CollectionsHandler.java:647)
	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:669)
	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:652)
	at org.apache.solr.handler.admin.CollectionsHandler.rejoinElection(CollectionsHandler.java:447)
	at org.apache.solr.handler.admin.CollectionsHandler.insurePreferredIsLeader(CollectionsHandler.java:362)
	at org.apache.solr.handler.admin.CollectionsHandler.handleBalanceLeaders(CollectionsHandler.java:269)
	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:235)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:782)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:282)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /zg/admin/collections. Reason:
<pre>    {trace=java.lang.AssertionError
	at org.apache.solr.common.util.ByteUtils.UTF8toUTF16(ByteUtils.java:36)
	at org.apache.solr.common.util.ByteUtils.UTF8toUTF16(ByteUtils.java:64)
	at org.apache.solr.common.cloud.ZkStateReader.fromJSON(ZkStateReader.java:137)
	at org.apache.solr.common.cloud.ZkNodeProps.load(ZkNodeProps.java:92)
	at org.apache.solr.cloud.DistributedQueue.containsTaskWithRequestId(DistributedQueue.java:127)
	at org.apache.solr.handler.admin.CollectionsHandler.overseerCollectionQueueContains(CollectionsHandler.java:647)
	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:669)
	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:652)
	at org.apache.solr.handler.admin.CollectionsHandler.rejoinElection(CollectionsHandler.java:447)
	at org.apache.solr.handler.admin.CollectionsHandler.insurePreferredIsLeader(CollectionsHandler.java:362)
	at org.apache.solr.handler.admin.CollectionsHandler.handleBalanceLeaders(CollectionsHandler.java:269)
	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:235)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:782)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:282)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:338)
	... 48 more




Build Log:
[...truncated 10082 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestRebalanceLeaders
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/init-core-data-001
   [junit4]   2> 1491198 T5962 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 1491198 T5962 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /zg/
   [junit4]   2> 1491203 T5962 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1491204 T5963 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1491304 T5962 oasc.ZkTestServer.run start zk server on port:20738
   [junit4]   2> 1491305 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1491306 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1491309 T5970 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40885ca name:ZooKeeperConnection Watcher:127.0.0.1:20738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1491309 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1491310 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1491310 T5962 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1491312 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1491313 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1491315 T5973 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c823094 name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1491315 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1491315 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1491316 T5962 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1491318 T5962 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1491319 T5962 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1491320 T5962 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1491322 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1491322 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1491325 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1491325 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1491427 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1491427 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1491429 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1491430 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1491431 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1491432 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1491433 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1491434 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1491436 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1491436 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1491438 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1491438 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1491440 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1491441 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1491442 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1491443 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1491444 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1491445 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1493135 T5962 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1
   [junit4]   2> 1493140 T5962 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1493145 T5962 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20743
   [junit4]   2> 1493145 T5962 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/zg, hostPort=20743, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores}
   [junit4]   2> 1493146 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1493146 T5962 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/'
   [junit4]   2> 1493205 T5962 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/solr.xml
   [junit4]   2> 1493272 T5962 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores
   [junit4]   2> 1493273 T5962 oasc.CoreContainer.<init> New CoreContainer 671212554
   [junit4]   2> 1493273 T5962 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/]
   [junit4]   2> 1493273 T5962 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/lib
   [junit4]   2> 1493274 T5962 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/lib).
   [junit4]   2> 1493283 T5962 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1493284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1493284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1493284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1493284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1493286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1493293 T5962 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1493294 T5962 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1493294 T5962 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1493295 T5962 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1493295 T5962 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20738/solr
   [junit4]   2> 1493295 T5962 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1493295 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1493296 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1493297 T5985 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e33214f name:ZooKeeperConnection Watcher:127.0.0.1:20738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1493298 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1493299 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1493301 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1493302 T5988 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6718d986 name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1493302 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1493303 T5962 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1493304 T5962 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1493305 T5962 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1493307 T5962 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1493308 T5962 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1493309 T5962 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1493310 T5962 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1493311 T5962 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1493312 T5962 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20743_zg
   [junit4]   2> 1493312 T5962 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20743_zg
   [junit4]   2> 1493313 T5962 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1493314 T5962 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1493315 T5962 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1493316 T5962 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:20743_zg
   [junit4]   2> 1493316 T5962 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1493317 T5962 oasc.Overseer.start Overseer (id=93617987373367299-127.0.0.1:20743_zg-n_0000000000) starting
   [junit4]   2> 1493318 T5962 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1493322 T5962 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1493323 T5990 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1493323 T5962 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1493323 T5989 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1493326 T5962 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores
   [junit4]   2> 1493328 T5962 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, dataDir=data/, shard=, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1, loadOnStartup=true, schema=schema.xml, transient=false, collection=control_collection, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/}
   [junit4]   2> 1493329 T5962 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/
   [junit4]   2> 1493329 T5962 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1493330 T5992 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1493330 T5992 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1493330 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1493330 T5992 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1493331 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:20743_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 1493331 T5989 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:20743_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1493332 T5989 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1493332 T5989 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1493333 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494331 T5992 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1494332 T5992 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1494332 T5992 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1494332 T5992 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1494333 T5992 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1494333 T5992 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/'
   [junit4]   2> 1494346 T5992 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1494348 T5992 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1494355 T5992 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1494370 T5992 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1494439 T5992 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1494540 T5992 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1494549 T5992 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1494886 T5992 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1494888 T5992 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1494890 T5992 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1494893 T5992 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1494905 T5992 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1494906 T5992 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1494906 T5992 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data/
   [junit4]   2> 1494906 T5992 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4bc15184
   [junit4]   2> 1494907 T5992 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data
   [junit4]   2> 1494907 T5992 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1494907 T5992 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1494907 T5992 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data/index
   [junit4]   2> 1494908 T5992 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=96.4521484375, floorSegmentMB=1.4296875, forceMergeDeletesPctAllowed=18.51724424491889, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1494908 T5992 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4e06e62a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26f0f76b),segFN=segments_1,generation=1}
   [junit4]   2> 1494909 T5992 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1494912 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1494914 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1494914 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1494914 T5992 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1494926 T5992 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494928 T5992 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494929 T5992 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494930 T5992 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494934 T5992 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json,/admin/plugins,/update/json/docs,/update/csv,/admin/ping,/config,/admin/luke,/admin/system,/admin/threads,/schema,/admin/file,/update,/admin/mbeans,/admin/logging,standard,/admin/segments,/replication,/get,/admin/properties
   [junit4]   2> 1494935 T5992 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1494936 T5992 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1494936 T5992 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1494937 T5992 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1494937 T5992 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1494938 T5992 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=15, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1494938 T5992 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4e06e62a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26f0f76b),segFN=segments_1,generation=1}
   [junit4]   2> 1494938 T5992 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1494939 T5992 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1268aa85[collection1] main
   [junit4]   2> 1494939 T5992 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1494941 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1494941 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1494941 T5992 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1494941 T5992 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1494942 T5993 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1268aa85[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1494943 T5992 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1494943 T5996 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20743/zg collection:control_collection shard:shard1
   [junit4]   2> 1494944 T5962 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 1494944 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1494944 T5996 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1494946 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1494947 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1494948 T5996 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1494948 T5999 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a1acfd1 name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1494949 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1494949 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1494950 T5962 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1494950 T5996 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1494950 T5996 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1494950 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> ASYNC  NEW_CORE C2726 name=collection1 org.apache.solr.core.SolrCore@46237e29 url=http://127.0.0.1:20743/zg/collection1 node=127.0.0.1:20743_zg C2726_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:20743/zg, node_name=127.0.0.1:20743_zg, core=collection1, state=down}
   [junit4]   2> 1494950 T5996 C:control_collection S:shard1 C2726 P20743 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20743/zg/collection1/
   [junit4]   2> 1494951 T5996 C:control_collection S:shard1 C2726 P20743 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1494951 T5996 C:control_collection S:shard1 C2726 P20743 oasc.SyncStrategy.syncToMe http://127.0.0.1:20743/zg/collection1/ has no replicas
   [junit4]   2> 1494951 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1494951 T5996 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20743/zg/collection1/ shard1
   [junit4]   2> 1494952 T5962 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1494952 T5996 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1494952 T5962 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1494952 T5999 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494952 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1494952 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494954 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1494956 T6002 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4cf39acf name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1494956 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1494956 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1494957 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1494958 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1494960 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1494960 T5989 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1494961 T5999 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494961 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494963 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1494963 T5989 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1494964 T5999 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494964 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1495008 T5996 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:20743/zg/collection1/ and leader is http://127.0.0.1:20743/zg/collection1/
   [junit4]   2> 1495009 T5996 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20743/zg
   [junit4]   2> 1495009 T5996 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1495009 T5996 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1495009 T5996 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1495011 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1495011 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:20743_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 1495012 T5989 oasco.ReplicaMutator.updateState Update state numShards=4 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:20743_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1495115 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1495115 T5999 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1496198 T5962 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1
   [junit4]   2> 1496200 T5962 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001
   [junit4]   2> 1496201 T5962 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1496203 T5962 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20767
   [junit4]   2> 1496204 T5962 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=20767, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores, hostContext=/zg, solrconfig=solrconfig.xml}
   [junit4]   2> 1496204 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1496205 T5962 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/'
   [junit4]   2> 1496236 T5962 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/solr.xml
   [junit4]   2> 1496273 T5962 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores
   [junit4]   2> 1496274 T5962 oasc.CoreContainer.<init> New CoreContainer 211441626
   [junit4]   2> 1496274 T5962 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/]
   [junit4]   2> 1496274 T5962 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/lib
   [junit4]   2> 1496275 T5962 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/lib).
   [junit4]   2> 1496284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1496285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1496285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1496285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1496285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1496287 T5962 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1496291 T5962 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1496292 T5962 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1496293 T5962 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1496293 T5962 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1496293 T5962 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20738/solr
   [junit4]   2> 1496293 T5962 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1496293 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1496294 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1496296 T6014 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@179bb877 name:ZooKeeperConnection Watcher:127.0.0.1:20738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1496297 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1496297 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1496299 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1496300 T6017 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9c04ef6 name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1496300 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1496303 T5962 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1497307 T5962 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20767_zg
   [junit4]   2> 1497308 T5962 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20767_zg
   [junit4]   2> 1497312 T5962 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1497314 T5962 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores
   [junit4]   2> 1497319 T5962 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, loadOnStartup=true, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1, dataDir=data/, shard=, coreNodeName=, collection=collection1, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/, config=solrconfig.xml}
   [junit4]   2> 1497319 T5962 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/
   [junit4]   2> 1497319 T5962 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1497321 T6018 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1497321 T6018 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1497323 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1497323 T6018 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1497323 T6018 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1497324 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20767_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1497324 T6018 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1497324 T5989 oasco.ReplicaMutator.updateState Update state numShards=4 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20767_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1497324 T5989 oasco.ReplicaMutator.updateState Collection already exists with numShards=4
   [junit4]   2> 1497325 T5989 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1497325 T6018 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1497325 T6018 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1497326 T5989 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1497326 T6017 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1497327 T6017 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1498325 T6018 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1498326 T6018 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1498326 T6018 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1498326 T6018 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1498327 T6018 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1498327 T6018 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1498340 T6018 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1498342 T6018 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1498349 T6018 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1498364 T6018 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1498399 T6018 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1498500 T6018 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1498508 T6018 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1498770 T6018 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1498771 T6018 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1498773 T6018 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1498776 T6018 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1498788 T6018 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1498789 T6018 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1498789 T6018 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1498789 T6018 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4bc15184
   [junit4]   2> 1498790 T6018 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1498791 T6018 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1498791 T6018 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1498791 T6018 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1498792 T6018 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=96.4521484375, floorSegmentMB=1.4296875, forceMergeDeletesPctAllowed=18.51724424491889, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1498793 T6018 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@47337b54 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4e5ec51),segFN=segments_1,generation=1}
   [junit4]   2> 1498793 T6018 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1498800 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1498800 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1498801 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1498801 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1498801 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1498802 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1498802 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1498802 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1498803 T6018 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1498822 T6018 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1498825 T6018 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1498827 T6018 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1498830 T6018 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1498837 T6018 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json,/admin/plugins,/update/json/docs,/update/csv,/admin/ping,/config,/admin/luke,/admin/system,/admin/threads,/schema,/admin/file,/update,/admin/mbeans,/admin/logging,standard,/admin/segments,/replication,/get,/admin/properties
   [junit4]   2> 1498839 T6018 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1498841 T6018 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1498841 T6018 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1498843 T6018 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1498843 T6018 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1498844 T6018 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=15, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1498845 T6018 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@47337b54 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4e5ec51),segFN=segments_1,generation=1}
   [junit4]   2> 1498845 T6018 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1498845 T6018 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@49167439[collection1] main
   [junit4]   2> 1498846 T6018 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1498846 T6018 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1498846 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1498847 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1498847 T6018 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1498847 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1498847 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1498848 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1498848 T6018 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1498848 T6018 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1498849 T6019 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@49167439[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1498849 T6018 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1498850 T6022 C:collection1 S:shard2 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20767/zg collection:collection1 shard:shard2
   [junit4]   2> 1498850 T5962 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 1498851 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1498851 T6022 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1498854 T6022 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1498855 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1498855 T6022 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1498856 T6022 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2727 name=collection1 org.apache.solr.core.SolrCore@2d6de342 url=http://127.0.0.1:20767/zg/collection1 node=127.0.0.1:20767_zg C2727_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:20767/zg, node_name=127.0.0.1:20767_zg, core=collection1, state=down}
   [junit4]   2> 1498856 T6022 C:collection1 S:shard2 C2727 P20767 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20767/zg/collection1/
   [junit4]   2> 1498856 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1498856 T6022 C:collection1 S:shard2 C2727 P20767 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1498857 T6022 C:collection1 S:shard2 C2727 P20767 oasc.SyncStrategy.syncToMe http://127.0.0.1:20767/zg/collection1/ has no replicas
   [junit4]   2> 1498857 T6022 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20767/zg/collection1/ shard2
   [junit4]   2> 1498857 T6022 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1498857 T5989 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1498858 T6017 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1498858 T6017 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1498860 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1498861 T5989 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1498861 T6017 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1498862 T6017 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1498863 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1498910 T6022 C:collection1 S:shard2 oasc.ZkController.register We are http://127.0.0.1:20767/zg/collection1/ and leader is http://127.0.0.1:20767/zg/collection1/
   [junit4]   2> 1498911 T6022 C:collection1 S:shard2 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20767/zg
   [junit4]   2> 1498911 T6022 C:collection1 S:shard2 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1498911 T6022 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1498911 T6022 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1498912 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1498913 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20767_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1498914 T5989 oasco.ReplicaMutator.updateState Update state numShards=4 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20767_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1498914 T6022 C:collection1 S:shard2 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1498914 T5989 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 1498915 T6017 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1498915 T6017 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1499784 T5962 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/cores/collection1
   [junit4]   2> 1499786 T5962 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001
   [junit4]   2> 1499787 T5962 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1499790 T5962 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20781
   [junit4]   2> 1499790 T5962 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/cores, hostPort=20781, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/tempDir-001/jetty2, hostContext=/zg}
   [junit4]   2> 1499790 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1499791 T5962 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/'
   [junit4]   2> 1499822 T5962 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/solr.xml
   [junit4]   2> 1499865 T5962 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/cores
   [junit4]   2> 1499866 T5962 oasc.CoreContainer.<init> New CoreContainer 1551576367
   [junit4]   2> 1499866 T5962 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/]
   [junit4]   2> 1499866 T5962 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/lib
   [junit4]   2> 1499867 T5962 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/lib).
   [junit4]   2> 1499877 T5962 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1499877 T5962 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1499879 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1499879 T5962 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1499879 T5962 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1499879 T5962 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1499885 T5962 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client w

[...truncated too long message...]

kStateReader.java:137)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkNodeProps.load(ZkNodeProps.java:92)
   [junit4]    > 	at org.apache.solr.cloud.DistributedQueue.containsTaskWithRequestId(DistributedQueue.java:127)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.overseerCollectionQueueContains(CollectionsHandler.java:647)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:669)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:652)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.rejoinElection(CollectionsHandler.java:447)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.insurePreferredIsLeader(CollectionsHandler.java:362)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.handleBalanceLeaders(CollectionsHandler.java:269)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:235)
   [junit4]    > 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:782)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:282)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
   [junit4]    > 	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
   [junit4]    > 	at org.eclipse.jetty.server.Server.handle(Server.java:368)
   [junit4]    > 	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
   [junit4]    > 	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
   [junit4]    > 	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
   [junit4]    > 	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
   [junit4]    > 	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
   [junit4]    > 	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
   [junit4]    > 	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
   [junit4]    > 	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > ,code=500}</pre></p>
   [junit4]    > <hr /><i><small>Powered by Jetty://</small></i>
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:338)
   [junit4]    > 	... 48 more
   [junit4]   2> 1592283 T5962 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=it_CH, timezone=Asia/Calcutta
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=211655760,total=402653184
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrQueryParserDefaultOperatorResource, TestNRTOpen, WordBreakSolrSpellCheckerTest, TestSolrXml, TestUpdate, TestCloudInspectUtil, TestElisionMultitermQuery, SpatialHeatmapFacetsTest, TestFieldCollectionResource, TestDistributedSearch, AnalysisErrorHandlingTest, TestLazyCores, DistribDocExpirationUpdateProcessorTest, CloudMLTQParserTest, DistributedFacetPivotLongTailTest, TestObjectReleaseTracker, TestSolrQueryParserResource, TestDistributedGrouping, FieldAnalysisRequestHandlerTest, ShardSplitTest, TestPostingsSolrHighlighter, DocValuesTest, FieldMutatingUpdateProcessorTest, TestManagedSynonymFilterFactory, PreAnalyzedFieldTest, DistributedQueryElevationComponentTest, TestInfoStreamLogging, HdfsChaosMonkeySafeLeaderTest, DistributedExpandComponentTest, OpenCloseCoreStressTest, BadCopyFieldTest, ClusterStateTest, TestStressLucene, CoreAdminCreateDiscoverTest, DeleteShardTest, HdfsBasicDistributedZkTest, TestSolrDeletionPolicy1, TestReload, ZkSolrClientTest, SpellCheckCollatorTest, AutoCommitTest, SpatialRPTFieldTypeTest, TestSuggestSpellingConverter, TestReplicaProperties, SharedFSAutoReplicaFailoverUtilsTest, CoreMergeIndexesAdminHandlerTest, TestOmitPositions, DirectSolrSpellCheckerTest, TestStressUserVersions, SpatialFilterTest, ZkStateWriterTest, DeleteReplicaTest, HdfsDirectoryTest, BadComponentTest, SortByFunctionTest, TestCustomSort, SolrPluginUtilsTest, TestCloudManagedSchemaConcurrent, TestCloudSchemaless, LeaderInitiatedRecoveryOnCommitTest, ShardRoutingTest, TestRebalanceLeaders]
   [junit4] Completed on J2 in 101.12s, 1 test, 1 error <<< FAILURES!

[...truncated 673 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1905 tests, 1 error, 62 ignored (22 assumptions)

Total time: 58 minutes 9 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 212847609 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2910 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2910/

1 tests failed.
REGRESSION:  org.apache.lucene.search.join.TestJoinUtil.testRandomOrdinalsJoin

Error Message:
expected:<296> but was:<269>

Stack Trace:
java.lang.AssertionError: expected:<296> but was:<269>
	at __randomizedtesting.SeedInfo.seed([5214ECCDEBB46393:EDC9ED543B48769F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.lucene.search.join.TestJoinUtil.assertTopDocs(TestJoinUtil.java:714)
	at org.apache.lucene.search.join.TestJoinUtil.testRandomOrdinalsJoin(TestJoinUtil.java:345)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 6851 lines...]
   [junit4] Suite: org.apache.lucene.search.join.TestJoinUtil
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestJoinUtil -Dtests.method=testRandomOrdinalsJoin -Dtests.seed=5214ECCDEBB46393 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=de_LU -Dtests.timezone=Europe/Jersey -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 0.76s J1 | TestJoinUtil.testRandomOrdinalsJoin <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<296> but was:<269>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5214ECCDEBB46393:EDC9ED543B48769F]:0)
   [junit4]    > 	at org.apache.lucene.search.join.TestJoinUtil.assertTopDocs(TestJoinUtil.java:714)
   [junit4]    > 	at org.apache.lucene.search.join.TestJoinUtil.testRandomOrdinalsJoin(TestJoinUtil.java:345)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {movieId=PostingsFormat(name=MockRandom), name=PostingsFormat(name=MockRandom), id=PostingsFormat(name=MockRandom), description=PostingsFormat(name=Memory doPackFST= true), type=PostingsFormat(name=LuceneVarGapFixedInterval), to=PostingsFormat(name=MockRandom), price=PostingsFormat(name=LuceneFixedGap), value=PostingsFormat(name=LuceneFixedGap), from=PostingsFormat(name=LuceneVarGapFixedInterval), subtitle=PostingsFormat(name=Memory doPackFST= true), productId=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{movieId=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Asserting), to=DocValuesFormat(name=Asserting), join_field=DocValuesFormat(name=Memory), idproductId=DocValuesFormat(name=Asserting), from=DocValuesFormat(name=Lucene50), productId=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=de_LU, timezone=Europe/Jersey
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=151345944,total=272105472
   [junit4]   2> NOTE: All tests run in this JVM: [TestJoinUtil]
   [junit4] Completed on J1 in 15.11s, 9 tests, 1 failure <<< FAILURES!

[...truncated 12 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/build.xml:456: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:2166: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/module-build.xml:58: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 4 suites, 29 tests, 1 failure

Total time: 18 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 62472 bytes
Compression is 0.0%
Took 0.27 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2909 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2909/

All tests passed

Build Log:
[...truncated 28423 lines...]
check-licenses:
     [echo] License check under: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jcl-over-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jcl-over-slf4j-1.7.6.jar.sha1

 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jul-to-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jul-to-slf4j-1.7.6.jar.sha1
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/slf4j-api-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/slf4j-api-1.7.6.jar.sha1

[...truncated 3 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:90: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:256: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
If you recently modified ivy-versions.properties or any module's ivy.xml,
make sure you run "ant clean-jars jar-checksums" before running precommit.

Total time: 93 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 1 artifacts
Archive block size is 32768
Received 0 blocks and 464 bytes
Compression is 0.0%
Took 0.2 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2908 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2908/

1 tests failed.
FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
java.net.SocketTimeoutException: Read timed out

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: java.net.SocketTimeoutException: Read timed out
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:774)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:277)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10147 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/init-core-data-001
   [junit4]   2> 1371559 T4624 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /nz_fss/la
   [junit4]   2> 1371564 T4624 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1371565 T4625 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1371666 T4624 oasc.ZkTestServer.run start zk server on port:25779
   [junit4]   2> 1371667 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1371668 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1371673 T4632 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e862073 name:ZooKeeperConnection Watcher:127.0.0.1:25779 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1371673 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1371674 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1371674 T4624 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1371679 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1371680 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1371681 T4635 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@268cfa3e name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1371681 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1371682 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1371682 T4624 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1371685 T4624 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1371686 T4624 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1371687 T4624 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1371689 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1371689 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1371692 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1371693 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1371796 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1371797 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1371798 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1371799 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1371801 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1371801 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1371803 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1371804 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1371805 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1371806 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1371807 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1371808 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1371810 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1371810 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1371812 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1371812 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1371814 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1371815 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1373224 T4624 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1
   [junit4]   2> 1373227 T4624 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1373233 T4624 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25783
   [junit4]   2> 1373233 T4624 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/nz_fss/la, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/tempDir-001/control/data, hostPort=25782, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores}
   [junit4]   2> 1373233 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1373234 T4624 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/'
   [junit4]   2> 1373272 T4624 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/solr.xml
   [junit4]   2> 1373316 T4624 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores
   [junit4]   2> 1373316 T4624 oasc.CoreContainer.<init> New CoreContainer 1448812155
   [junit4]   2> 1373316 T4624 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/]
   [junit4]   2> 1373316 T4624 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/lib
   [junit4]   2> 1373317 T4624 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/lib).
   [junit4]   2> 1373326 T4624 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1373327 T4624 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1373327 T4624 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1373327 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1373327 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1373329 T4624 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1373335 T4624 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1373335 T4624 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1373336 T4624 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1373336 T4624 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1373336 T4624 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25779/solr
   [junit4]   2> 1373336 T4624 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1373337 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1373337 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1373339 T4647 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@772c4729 name:ZooKeeperConnection Watcher:127.0.0.1:25779 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1373340 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1373340 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1373342 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1373343 T4650 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3548fe79 name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1373343 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1373345 T4624 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1373346 T4624 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1373348 T4624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1373349 T4624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1373350 T4624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1373351 T4624 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1373352 T4624 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1373353 T4624 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1373354 T4624 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25782_nz_fss%2Fla
   [junit4]   2> 1373354 T4624 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25782_nz_fss%2Fla
   [junit4]   2> 1373355 T4624 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1373356 T4624 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1373357 T4624 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1373358 T4624 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:25782_nz_fss%2Fla
   [junit4]   2> 1373358 T4624 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1373359 T4624 oasc.Overseer.start Overseer (id=93615144876572675-127.0.0.1:25782_nz_fss%2Fla-n_0000000000) starting
   [junit4]   2> 1373361 T4624 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1373364 T4624 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1373365 T4652 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1373365 T4651 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1373365 T4624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1373369 T4624 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores
   [junit4]   2> 1373371 T4624 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/, transient=false, config=solrconfig.xml, coreNodeName=, loadOnStartup=true, shard=, dataDir=data/, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1, schema=schema.xml, name=collection1}
   [junit4]   2> 1373371 T4624 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/
   [junit4]   2> 1373371 T4624 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1373372 T4654 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1373372 T4654 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1373373 T4654 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1373373 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1373373 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25782_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 1373374 T4651 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25782_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1373374 T4651 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1373374 T4651 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1373375 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1374374 T4654 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1374375 T4654 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1374375 T4654 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1374375 T4654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1374376 T4654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1374376 T4654 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/'
   [junit4]   2> 1374390 T4654 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1374392 T4654 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1374400 T4654 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1374419 T4654 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1374457 T4654 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1374559 T4654 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1374567 T4654 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1374890 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1374903 T4654 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1374905 T4654 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1374912 T4654 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1374915 T4654 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1374919 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1374919 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1374919 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1374919 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1374920 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1374920 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1374920 T4654 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1374920 T4654 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1374921 T4654 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data/
   [junit4]   2> 1374921 T4654 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6ba9dace
   [junit4]   2> 1374921 T4654 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data
   [junit4]   2> 1374922 T4654 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1374922 T4654 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1374922 T4654 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data/index
   [junit4]   2> 1374922 T4654 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=23.0185546875, floorSegmentMB=1.162109375, forceMergeDeletesPctAllowed=11.77904699044102, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1374923 T4654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@763fe6fe lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5d566a58),segFN=segments_1,generation=1}
   [junit4]   2> 1374923 T4654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1374927 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1374927 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1374927 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1374929 T4654 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1374951 T4654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1374953 T4654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1374954 T4654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1374956 T4654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1374960 T4654 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/admin/luke,/admin/segments,/admin/file,/admin/threads,/admin/ping,/update,/admin/mbeans,/admin/logging,/admin/properties,/update/csv,/config,/replication,/get,/schema,standard,/update/json,/admin/system,/update/json/docs
   [junit4]   2> 1374962 T4654 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1374963 T4654 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1374963 T4654 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1374965 T4654 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1374965 T4654 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1374965 T4654 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=69.14453125, floorSegmentMB=1.6064453125, forceMergeDeletesPctAllowed=4.444172208952494, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4601051453371784
   [junit4]   2> 1374966 T4654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@763fe6fe lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5d566a58),segFN=segments_1,generation=1}
   [junit4]   2> 1374966 T4654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1374966 T4654 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1ea1aa6a[collection1] main
   [junit4]   2> 1374966 T4654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1374967 T4654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1374967 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1374967 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1374969 T4654 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1374969 T4654 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1374969 T4655 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1ea1aa6a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1374970 T4658 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25782/nz_fss/la collection:control_collection shard:shard1
   [junit4]   2> 1374970 T4624 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1374970 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1374970 T4658 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1374972 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1374973 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1374974 T4658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1374974 T4662 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5aedef0c name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1374974 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1374975 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1374975 T4624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1374975 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1374976 T4658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1374976 T4658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C871 name=collection1 org.apache.solr.core.SolrCore@7aad97f2 url=http://127.0.0.1:25782/nz_fss/la/collection1 node=127.0.0.1:25782_nz_fss%2Fla C871_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:25782_nz_fss%2Fla, base_url=http://127.0.0.1:25782/nz_fss/la, state=down}
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 C871 P25782 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25782/nz_fss/la/collection1/
   [junit4]   2> 1374977 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 C871 P25782 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 C871 P25782 oasc.SyncStrategy.syncToMe http://127.0.0.1:25782/nz_fss/la/collection1/ has no replicas
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25782/nz_fss/la/collection1/ shard1
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1374978 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1374978 T4662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1374978 T4624 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1374979 T4624 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1374980 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1374980 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1374981 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1374982 T4665 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1cde29fe name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1374982 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1374982 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1374982 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1374985 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1374986 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1374986 T4651 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1374987 T4662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1374987 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1375032 T4658 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:25782/nz_fss/la/collection1/ and leader is http://127.0.0.1:25782/nz_fss/la/collection1/
   [junit4]   2> 1375033 T4658 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25782/nz_fss/la
   [junit4]   2> 1375033 T4658 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1375033 T4658 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1375033 T4658 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1375034 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1375034 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25782_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 1375035 T4651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25782_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1375035 T4651 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1375036 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1375036 T4662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1375139 T4662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1375139 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1376129 T4624 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1
   [junit4]   2> 1376131 T4624 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001
   [junit4]   2> 1376132 T4624 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1376134 T4624 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25795
   [junit4]   2> 1376135 T4624 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/nz_fss/la, solrconfig=solrconfig.xml, hostPort=25794, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/tempDir-001/jetty1}
   [junit4]   2> 1376135 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1376135 T4624 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/'
   [junit4]   2> 1376167 T4624 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/solr.xml
   [junit4]   2> 1376203 T4624 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores
   [junit4]   2> 1376203 T4624 oasc.CoreContainer.<init> New CoreContainer 1379001132
   [junit4]   2> 1376203 T4624 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/]
   [junit4]   2> 1376204 T4624 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/lib
   [junit4]   2> 1376204 T4624 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/lib).
   [junit4]   2> 1376214 T4624 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1376214 T4624 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1376214 T4624 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1376216 T4624 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1376216 T4624 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1376216 T4624 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1376221 T4624 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1376222 T4624 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1376222 T4624 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1376222 T4624 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1376222 T4624 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25779/solr
   [junit4]   2> 1376222 T4624 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1376223 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1376223 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1376225 T4677 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ded99f0 name:ZooKeeperConnection Watcher:127.0.0.1:25779 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1376226 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1376226 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1376229 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1376230 T4680 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@48297555 name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1376230 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1376243 T4624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1377247 T4624 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25794_nz_fss%2Fla
   [junit4]   2> 1377248 T4624 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25794_nz_fss%2Fla
   [junit4]   2> 1377251 T4624 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1377253 T4624 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores
   [junit4]   2> 1377255 T4624 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, dataDir=data/, config=solrconfig.xml, name=collection1, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/, loadOnStartup=true, coreNodeName=, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1, shard=}
   [junit4]   2> 1377255 T4624 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/
   [junit4]   2> 1377255 T4624 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1377257 T4681 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1377257 T4681 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1377258 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1377258 T4681 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1377259 T4681 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1377259 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25794_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1377259 T4681 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1377259 T4651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25794_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1377259 T4651 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 1377259 T4651 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1377260 T4681 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1377260 T4681 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1377260 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1377261 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1377261 T4680 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1378261 T4681 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1378262 T4681 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1378262 T4681 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1378262 T4681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1378263 T4681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1378263 T4681 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1378277 T4681 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1378279 T4681 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1378288 T4681 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1378306 T4681 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1378344 T4681 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1378446 T4681 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1378454 T4681 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1378741 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1378755 T4681 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1378757 T4681 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1378763 T4681 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1378766 T4681 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1378769 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1378770 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1378770 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1378770 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1378772 T4681 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6ba9dace
   [junit4]   2> 1378772 T4681 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1378772 T4681 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1378773 T4681 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1378773 T4681 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1378773 T4681 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=23.0185546875, floorSegmentMB=1.162109375, forceMergeDeletesPctAllowed=11.77904699044102, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1378774 T4681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@56774af0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31d8fee9),segFN=segments_1,generation=1}
   [junit4]   2> 1378774 T4681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1378778 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1378780 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1378780 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1378780 T4681 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1378797 T4681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1378799 T4681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1378801 T4681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1378802 T4681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1378807 T4681 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/admin/luke,/admin/segments,/admin/file,/admin/threads,/admin/ping,/update,/admin/mbeans,/admin/logging,/admin/properties,/update/csv,/config,/replication,/get,/schema,standard,/update/json,/admin/system,/update/json/docs
   [junit4]   2> 1378809 T4681 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1378810 T4681 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1378811 T4681 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1378812 T4681 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1378812 T4681 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1378813 T4681 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=69.14453125, floorSegmentMB=1.6064453125, forceMergeDeletesPctAllowed=4.444172208952494, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4601051453371784
   [junit4]   2> 1378813 T4681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@56774af0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31d8fee9),segFN=segments_1,generation=1}
   [junit4]   2> 1378813 T4681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1378814 T4681 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7a6a994e[collection1] main
   [junit4]   2> 1378814 T4681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1378814 T4681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1378814 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1378815 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1378815 T4681 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1378815 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1378815 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1378816 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1378816 T4681 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1378816 T4681 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1378817 T4682 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a6a994e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1378817 T4681 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1378818 T4685 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25794/nz_fss/la collection:collection1 shard:shard1
   [junit4]   2> 1378818 T4624 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1378818 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1378818 T4685 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1378821 T4685 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1378822 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1378823 T4685 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1378823 T4685 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C872 name=collection1 org.apache.solr.core.SolrCore@1bffbe5b url=http://127.0.0.1:25794/nz_fss/la/collection1 node=127.0.0.1:25794_nz_fss%2Fla C872_STATE=coll:collection1 core:collection1 props:{core=collection1, node_name=127.0.0.1:25794_nz_fss%2Fla, base_url=http://127.0.0.1:25794/nz_fss/la, state=down}
   [junit4]   2> 1378823 T4685 C:collection1 S:shard1 C872 P25794 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25794/nz_fss/la/collection1/
   [junit4]   2> 1378823 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1378823 T4685 C:collection1 S:shard1 C872 P25794 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1378824 T4685 C:collection1 S:shard1 C872 P25794 oasc.SyncStrategy.syncToMe http://127.0.0.1:25794/nz_fss/la/collection1/ has no replicas
   [junit4]   2> 1378824 T4685 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25794/nz_fss/la/collection1/ shard1
   [junit4]   2> 1378824 T4685 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1378824 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1378825 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1378825 T4680 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1378827 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1378827 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1378828 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1378828 T4680 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1378829 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1378878 T4685 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:25794/nz_fss/la/collection1/ and leader is http://127.0.0.1:25794/nz_fss/la/collection1/
   [junit4]   2> 1378879 T4685 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25794/nz_fss/la
   [junit4]   2> 1378879 T4685 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1378879 T4685 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1378879 T4685 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1378880 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1378880 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25794_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1378881 T4651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25794_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1378881 T4685 C:collection1 S:shard1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1378882 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 1378882 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1378883 T4680 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1379746 T4624 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores/collection1
   [junit4]   2> 1379748 T4624 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001
   [junit4]   2> 1379749 T4624 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1379752 T4624 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25804
   [junit4]   2> 1379752 T4624 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores, hostPort=25803, hostContext=/nz_fss/la}
   [junit4]   2> 1379753 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1379753 T4624 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/'
   [junit4]   2> 1379782 T4624 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/solr.xml
   [junit4]   2> 1379816 T4624 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores
   [junit4]   2> 1379817 T4624 oasc.CoreContainer.<init> New CoreContainer 1199359807
   [junit4]   2> 1379817 T4624 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/]
   [junit4]   2> 1379817 T4624 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/lib
   [junit4]   2> 1379818 T4624 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/lib).
   [junit4]   2> 1379827 T4624 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1379827 T4624 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1379827 T4624 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1379827 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1379829 T4624 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1379829 T4624 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1379833 T4624 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1379834 T4624 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1379834 T4624 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1379834 T4624 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1379835 T4624 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25779/solr
   [junit4]   2> 1379835 T4624 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1379835 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1379836 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1379837 T4698 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ca4f3b4 name:ZooKeeperConnection Watcher:127.0.0.1:25779 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1379838 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1379838 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1379840 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1379841 T4701 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e7c6687 name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1379841 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1379845 T4624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1380848 T4624 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25803_nz_fss%2Fla
   [junit4]   2> 1380849 T4624 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25803_nz_fss%2Fla
   [junit4]   2> 1380852 T4624 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1380855 T4624 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores
   [junit4]   2> 1380857 T4624 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores/collection1/, dataDir=data/, name=collection1, shard=, schema=schema.xml, transient=false, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores/collection1, config=solrconfig.xml}
   [junit4]   2> 1380857 T4624 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores/collection1/
   [junit4]   2> 1380858 T4624 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1380859 T4702 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1380859 T4702 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1380860 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1380861 T4702 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1380861 T4702 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1380861 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25803_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25803/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1380861 T4702 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1380862 T4651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25803_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25803/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1380862 T4651 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 1380862 T4651 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1380862 T4702 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1380862 T4702 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1380863 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 4
   [junit4]   2> 1380863 T4701 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1380863 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1380864 T4701 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 1380864 T4680 oascc.ZkStateReader.update

[...truncated too long message...]

.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4829, name=updateExecutor-653-thread-6, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@282848d9 rejected from java.util.concurrent.ThreadPoolExecutor@e71bb73[Shutting down, pool size = 4, active threads = 4, queued tasks = 0, completed tasks = 46]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BFFCE504F86C54DD]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4832, name=updateExecutor-653-thread-7, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@5b761ce3 rejected from java.util.concurrent.ThreadPoolExecutor@e71bb73[Shutting down, pool size = 3, active threads = 3, queued tasks = 0, completed tasks = 47]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BFFCE504F86C54DD]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4835, name=updateExecutor-653-thread-8, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@f2d440b rejected from java.util.concurrent.ThreadPoolExecutor@e71bb73[Shutting down, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 48]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BFFCE504F86C54DD]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4822, name=updateExecutor-653-thread-4, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@ad15bd2 rejected from java.util.concurrent.ThreadPoolExecutor@e71bb73[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 49]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BFFCE504F86C54DD]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1955257 T4624 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=PostingsFormat(name=Asserting), timestamp=PostingsFormat(name=LuceneFixedGap), id=PostingsFormat(name=Memory doPackFST= true), a_t=PostingsFormat(name=LuceneFixedGap), intDefault=PostingsFormat(name=Asserting), range_facet_l=PostingsFormat(name=LuceneFixedGap), text=PostingsFormat(name=LuceneVarGapDocFreqInterval), multiDefault=PostingsFormat(name=LuceneFixedGap)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=DefaultSimilarity, locale=ar_OM, timezone=Europe/Tiraspol
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=78560712,total=365953024
   [junit4]   2> NOTE: All tests run in this JVM: [TestSystemIdResolver, TestSchemaSimilarityResource, PolyFieldTest, CoreAdminCreateDiscoverTest, PrimitiveFieldTypeTest, RegexBoostProcessorTest, PreAnalyzedUpdateProcessorTest, HttpPartitionTest, PreAnalyzedFieldTest, TestSolrQueryParserResource, IndexSchemaRuntimeFieldTest, TestArbitraryIndexDir, TestReload, NumericFieldsTest, TestJmxMonitoredMap, TestBulkSchemaAPI, TestManagedResource, TestSolrQueryParserDefaultOperatorResource, TestFieldTypeResource, AnalysisAfterCoreReloadTest, DocValuesMissingTest, DistributedMLTComponentTest, EchoParamsTest, TermVectorComponentDistributedTest, TestCloudInspectUtil, QueryResultKeyTest, TestDistributedGrouping, HdfsBasicDistributedZkTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestCollationField, TestLRUCache, TestSolrDeletionPolicy1, TestExtendedDismaxParser, TestTrackingShardHandlerFactory, SolrInfoMBeanTest, BlockDirectoryTest, XsltUpdateRequestHandlerTest, UnloadDistributedZkTest, TestBinaryField, DirectUpdateHandlerTest, CollectionReloadTest, SimpleMLTQParserTest, TestUniqueKeyFieldResource, TestDynamicLoading, TestLeaderElectionZkExpiry, TestManagedSchemaDynamicFieldResource, TestAddFieldRealTimeGet, TestDistribDocBasedVersion, DistributedIntervalFacetingTest, TestJsonFacets, UpdateParamsTest, HdfsSyncSliceTest, TestFastLRUCache, IndexSchemaTest, FieldAnalysisRequestHandlerTest, TestManagedSchemaFieldTypeResource, TestConfig, TestSolrJ, CollectionsAPIDistributedZkTest, BasicDistributedZkTest, MultiThreadedOCPTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 583.73s, 1 test, 1 error <<< FAILURES!

[...truncated 614 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1905 tests, 1 error, 61 ignored (21 assumptions)

Total time: 60 minutes 33 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218537067 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2907 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2907/

1 tests failed.
FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
java.net.SocketTimeoutException: Read timed out

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: java.net.SocketTimeoutException: Read timed out
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:774)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:277)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9900 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/init-core-data-001
   [junit4]   2> 440254 T1875 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 440258 T1875 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 440258 T1876 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 440358 T1875 oasc.ZkTestServer.run start zk server on port:14242
   [junit4]   2> 440359 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 440360 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 440362 T1883 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29b8b880 name:ZooKeeperConnection Watcher:127.0.0.1:14242 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 440363 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 440363 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 440363 T1875 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 440365 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 440366 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 440367 T1886 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3bb633cd name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 440367 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 440367 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 440367 T1875 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 440368 T1875 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 440369 T1875 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 440370 T1875 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 440372 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 440372 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 440374 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 440374 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 440475 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 440476 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 440477 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 440477 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 440479 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 440479 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 440480 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 440481 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 440482 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 440482 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 440483 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 440484 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 440485 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 440485 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 440486 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 440487 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 440488 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 440488 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 441510 T1875 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1
   [junit4]   2> 441513 T1875 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 441517 T1875 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14246
   [junit4]   2> 441517 T1875 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=14245, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores}
   [junit4]   2> 441518 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 441518 T1875 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/'
   [junit4]   2> 441547 T1875 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/solr.xml
   [junit4]   2> 441599 T1875 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores
   [junit4]   2> 441600 T1875 oasc.CoreContainer.<init> New CoreContainer 1380014511
   [junit4]   2> 441600 T1875 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/]
   [junit4]   2> 441601 T1875 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/lib
   [junit4]   2> 441602 T1875 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/lib).
   [junit4]   2> 441611 T1875 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 441611 T1875 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 441613 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 441613 T1875 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 441613 T1875 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 441613 T1875 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 441618 T1875 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 441619 T1875 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 441619 T1875 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 441619 T1875 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 441619 T1875 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14242/solr
   [junit4]   2> 441619 T1875 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 441620 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 441620 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 441622 T1898 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@994d153 name:ZooKeeperConnection Watcher:127.0.0.1:14242 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 441623 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 441623 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 441624 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 441625 T1901 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fcb98d6 name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 441626 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 441627 T1875 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 441628 T1875 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 441629 T1875 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 441631 T1875 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 441632 T1875 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 441633 T1875 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 441634 T1875 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 441635 T1875 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 441636 T1875 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14245_
   [junit4]   2> 441636 T1875 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14245_
   [junit4]   2> 441637 T1875 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 441638 T1875 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 441639 T1875 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 441640 T1875 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:14245_
   [junit4]   2> 441640 T1875 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 441641 T1875 oasc.Overseer.start Overseer (id=93614406352961539-127.0.0.1:14245_-n_0000000000) starting
   [junit4]   2> 441642 T1875 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 441646 T1875 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 441647 T1903 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 441647 T1902 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 441647 T1875 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 441650 T1875 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores
   [junit4]   2> 441651 T1875 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1, loadOnStartup=true, dataDir=data/, transient=false, coreNodeName=, schema=schema.xml, config=solrconfig.xml, collection=control_collection, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/}
   [junit4]   2> 441652 T1875 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/
   [junit4]   2> 441652 T1875 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 441653 T1905 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 441653 T1905 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 441653 T1905 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 441653 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 441654 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:14245_",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 441655 T1902 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:14245_",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 441655 T1902 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 441655 T1902 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 441656 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 442654 T1905 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 442655 T1905 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 442655 T1905 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 442656 T1905 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 442656 T1905 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 442656 T1905 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/'
   [junit4]   2> 442674 T1905 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 442676 T1905 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 442686 T1905 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 442709 T1905 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 442761 T1905 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 442862 T1905 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 442873 T1905 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 443284 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 443301 T1905 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 443304 T1905 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 443314 T1905 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 443318 T1905 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 443322 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 443323 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 443323 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 443323 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 443324 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 443324 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 443325 T1905 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 443325 T1905 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 443325 T1905 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data/
   [junit4]   2> 443326 T1905 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@42d7ecc1
   [junit4]   2> 443326 T1905 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data
   [junit4]   2> 443327 T1905 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data/index/
   [junit4]   2> 443327 T1905 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 443327 T1905 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data/index
   [junit4]   2> 443328 T1905 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6542973963737145]
   [junit4]   2> 443329 T1905 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@45455d46 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ceec231),segFN=segments_1,generation=1}
   [junit4]   2> 443329 T1905 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 443334 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 443335 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 443335 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 443335 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 443336 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 443336 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 443336 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 443337 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 443337 T1905 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 443357 T1905 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 443360 T1905 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 443362 T1905 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 443364 T1905 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 443369 T1905 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/ping,/admin/luke,/admin/threads,/admin/properties,/admin/segments,/get,/admin/mbeans,/schema,/update/json/docs,/config,/update/csv,/admin/logging,/admin/plugins,/update,/update/json,/replication,/admin/system,standard
   [junit4]   2> 443371 T1905 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 443373 T1905 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 443373 T1905 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 443374 T1905 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 443375 T1905 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 443375 T1905 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=57.2275390625, floorSegmentMB=0.2529296875, forceMergeDeletesPctAllowed=20.90666064053401, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 443376 T1905 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@45455d46 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ceec231),segFN=segments_1,generation=1}
   [junit4]   2> 443376 T1905 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 443377 T1905 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@73d0e76e[collection1] main
   [junit4]   2> 443377 T1905 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 443377 T1905 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 443378 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 443378 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 443378 T1905 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 443378 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 443379 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 443379 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 443379 T1905 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 443380 T1905 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 443380 T1905 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 443381 T1906 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@73d0e76e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 443381 T1909 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:14245 collection:control_collection shard:shard1
   [junit4]   2> 443381 T1875 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 443382 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 443382 T1909 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 443385 T1909 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 443386 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 443386 T1909 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 443386 T1909 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 443386 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 443386 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C206 name=collection1 org.apache.solr.core.SolrCore@1ab3596b url=http://127.0.0.1:14245/collection1 node=127.0.0.1:14245_ C206_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:14245, core=collection1, node_name=127.0.0.1:14245_}
   [junit4]   2> 443386 T1909 C:control_collection S:shard1 C206 P14245 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:14245/collection1/
   [junit4]   2> 443387 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 443387 T1909 C:control_collection S:shard1 C206 P14245 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 443388 T1909 C:control_collection S:shard1 C206 P14245 oasc.SyncStrategy.syncToMe http://127.0.0.1:14245/collection1/ has no replicas
   [junit4]   2> 443388 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 443388 T1909 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:14245/collection1/ shard1
   [junit4]   2> 443388 T1909 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 443389 T1913 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f02e185 name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 443389 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 443389 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 443390 T1875 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 443391 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 443392 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 443392 T1875 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 443494 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 443494 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 443544 T1909 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:14245/collection1/ and leader is http://127.0.0.1:14245/collection1/
   [junit4]   2> 443545 T1909 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:14245
   [junit4]   2> 443545 T1909 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 443545 T1909 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 443545 T1909 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 443546 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 443546 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:14245_",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 443547 T1902 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:14245_",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 443650 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 443650 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 444808 T1875 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1
   [junit4]   2> 444811 T1875 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001
   [junit4]   2> 444812 T1875 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 444815 T1875 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14251
   [junit4]   2> 444815 T1875 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostPort=14250}
   [junit4]   2> 444816 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 444816 T1875 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/'
   [junit4]   2> 444856 T1875 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/solr.xml
   [junit4]   2> 444896 T1875 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores
   [junit4]   2> 444896 T1875 oasc.CoreContainer.<init> New CoreContainer 1555043384
   [junit4]   2> 444897 T1875 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/]
   [junit4]   2> 444897 T1875 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/lib
   [junit4]   2> 444897 T1875 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/lib).
   [junit4]   2> 444906 T1875 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 444907 T1875 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 444907 T1875 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 444907 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 444907 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 444909 T1875 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 444914 T1875 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 444915 T1875 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 444915 T1875 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 444915 T1875 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 444915 T1875 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14242/solr
   [junit4]   2> 444916 T1875 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 444916 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 444917 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 444918 T1925 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3fa3586c name:ZooKeeperConnection Watcher:127.0.0.1:14242 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 444919 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 444920 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 444921 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 444922 T1928 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@128b39c7 name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 444922 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 444925 T1875 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 445928 T1875 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14250_
   [junit4]   2> 445929 T1875 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14250_
   [junit4]   2> 445932 T1875 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 445934 T1875 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores
   [junit4]   2> 445935 T1875 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, shard=, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/, config=solrconfig.xml, dataDir=data/, collection=collection1, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1, name=collection1, schema=schema.xml}
   [junit4]   2> 445936 T1875 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/
   [junit4]   2> 445936 T1875 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 445937 T1929 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 445937 T1929 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 445937 T1929 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 445937 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 445938 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14250_",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 445938 T1902 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14250_",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 445939 T1902 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 445939 T1902 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 445940 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 445940 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 445940 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 446938 T1929 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 446939 T1929 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 446939 T1929 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 446939 T1929 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 446940 T1929 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 446940 T1929 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/'
   [junit4]   2> 446954 T1929 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 446956 T1929 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 446965 T1929 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 446982 T1929 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 447020 T1929 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 447121 T1929 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 447129 T1929 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 447470 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 447482 T1929 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 447484 T1929 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 447492 T1929 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 447495 T1929 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 447498 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 447498 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 447498 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 447498 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 447499 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 447499 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 447499 T1929 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 447499 T1929 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 447500 T1929 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 447500 T1929 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@42d7ecc1
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6542973963737145]
   [junit4]   2> 447502 T1929 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@a1b712e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@fa38831),segFN=segments_1,generation=1}
   [junit4]   2> 447502 T1929 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 447507 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 447507 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 447507 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 447508 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 447508 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 447508 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 447508 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 447509 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 447509 T1929 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 447523 T1929 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 447524 T1929 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 447526 T1929 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 447528 T1929 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 447533 T1929 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/ping,/admin/luke,/admin/threads,/admin/properties,/admin/segments,/get,/admin/mbeans,/schema,/update/json/docs,/config,/update/csv,/admin/logging,/admin/plugins,/update,/update/json,/replication,/admin/system,standard
   [junit4]   2> 447535 T1929 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 447536 T1929 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 447536 T1929 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 447537 T1929 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 447537 T1929 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 447539 T1929 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=57.2275390625, floorSegmentMB=0.2529296875, forceMergeDeletesPctAllowed=20.90666064053401, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 447539 T1929 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@a1b712e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@fa38831),segFN=segments_1,generation=1}
   [junit4]   2> 447539 T1929 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 447540 T1929 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@57b843f7[collection1] main
   [junit4]   2> 447540 T1929 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 447540 T1929 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 447541 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 447541 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 447541 T1929 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 447541 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 447542 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 447542 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 447542 T1929 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 447542 T1929 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 447543 T1930 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@57b843f7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 447543 T1929 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 447544 T1933 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:14250 collection:collection1 shard:shard3
   [junit4]   2> 447544 T1875 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 447544 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 447545 T1933 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 447548 T1933 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 447549 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C207 name=collection1 org.apache.solr.core.SolrCore@ab3b34e url=http://127.0.0.1:14250/collection1 node=127.0.0.1:14250_ C207_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:14250, core=collection1, node_name=127.0.0.1:14250_}
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 C207 P14250 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:14250/collection1/
   [junit4]   2> 447549 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 C207 P14250 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 C207 P14250 oasc.SyncStrategy.syncToMe http://127.0.0.1:14250/collection1/ has no replicas
   [junit4]   2> 447550 T1933 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:14250/collection1/ shard3
   [junit4]   2> 447550 T1933 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 447550 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447550 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447550 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447553 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 447553 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 447655 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447655 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447655 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447705 T1933 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:14250/collection1/ and leader is http://127.0.0.1:14250/collection1/
   [junit4]   2> 447706 T1933 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:14250
   [junit4]   2> 447706 T1933 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 447706 T1933 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 447706 T1933 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 447707 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 447708 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14250_",
   [junit4]   2> 	  "shard":"shard3"} current state version: 7
   [junit4]   2> 447709 T1902 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14250_",
   [junit4]   2> 	  "shard":"shard3"}
   [junit4]   2> 447811 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447811 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447811 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 448664 T1875 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1
   [junit4]   2> 448666 T1875 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001
   [junit4]   2> 448668 T1875 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 448670 T1875 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14256
   [junit4]   2> 448670 T1875 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=14255, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/tempDir-001/jetty2}
   [junit4]   2> 448671 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 448671 T1875 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/'
   [junit4]   2> 448720 T1875 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/solr.xml
   [junit4]   2> 448781 T1875 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores
   [junit4]   2> 448781 T1875 oasc.CoreContainer.<init> New CoreContainer 2041762435
   [junit4]   2> 448782 T1875 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/]
   [junit4]   2> 448782 T1875 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/lib
   [junit4]   2> 448783 T1875 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/lib).
   [junit4]   2> 448797 T1875 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 448797 T1875 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 448798 T1875 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 448798 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 448798 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 448799 T1875 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 448799 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 448799 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 448800 T1875 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 448800 T1875 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 448800 T1875 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 448807 T1875 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 448808 T1875 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 448809 T1875 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 448809 T1875 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 448809 T1875 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14242/solr
   [junit4]   2> 448810 T1875 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 448810 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 448811 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 448813 T1946 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@657aa9f6 name:ZooKeeperConnection Watcher:127.0.0.1:14242 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 448814 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 448814 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 448816 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 448817 T1949 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14f9a04e name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 448818 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 448822 T1875 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 449825 T1875 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14255_
   [junit4]   2> 449826 T1875 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14255_
   [junit4]   2> 449829 T1875 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 449831 T1875 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores
   [junit4]   2> 449832 T1875 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, shard=, schema=schema.xml, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1, name=collection1, collection=collection1, config=solrconfig.xml, loadOnStartup=true, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1/}
   [junit4]   2> 449833 T1875 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1/
   [junit4]   2> 449833 T1875 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 449834 T1950 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 449834 T1950 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 449834 T1950 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 449834 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 449835 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14255",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14255_",
   [junit4]   2> 	  "shard":null} current state version: 8
   [junit4]   2> 449836 T1902 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14255",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14255_",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 449836 T1902 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 449836 T1902 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 449837 T1949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 449837 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 449837 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 449837 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 450835 T1950 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 450836 T1950 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 450836 T1950 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 450837 T1950 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 450837 T1950 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 450837 T1950 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1/'
   [junit4]   2> 450855 T1950 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 450857 T1950 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 450868 T1950 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 450891 T1950 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 450941 T1950 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 451042 T1950 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 451053 T1950 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 451470 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 451486 T1950 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 451489 T1950 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 451499 T1950 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 451504 T1950 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 451508 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 451509 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 451510 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 451510 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 451511 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 451511 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 451511 T1950 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 451512 T1950 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 451512 T1950 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/worksp

[...truncated too long message...]

ent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 5, active threads = 5, queued tasks = 0, completed tasks = 48]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2070, name=updateExecutor-278-thread-5, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@350152a3 rejected from java.util.concurrent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 4, active threads = 4, queued tasks = 0, completed tasks = 49]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2073, name=updateExecutor-278-thread-6, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@162ec62c rejected from java.util.concurrent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 3, active threads = 3, queued tasks = 0, completed tasks = 50]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2076, name=updateExecutor-278-thread-7, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@743f2543 rejected from java.util.concurrent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 51]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2079, name=updateExecutor-278-thread-8, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@43cb4a37 rejected from java.util.concurrent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 52]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1012391 T1875 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001
   [junit4]   2> 572143 T1874 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {multiDefault=BlockTreeOrds(blocksize=128), a_t=PostingsFormat(name=Memory doPackFST= false), text=FST50, range_facet_l=PostingsFormat(name=Memory doPackFST= false), intDefault=PostingsFormat(name=Memory doPackFST= false), timestamp=PostingsFormat(name=Memory doPackFST= false), _version_=PostingsFormat(name=Memory doPackFST= false), id=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{timestamp=DocValuesFormat(name=SimpleText)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=en_IE, timezone=America/Kentucky/Monticello
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=171993136,total=303038464
   [junit4]   2> NOTE: All tests run in this JVM: [TestCoreContainer, ChaosMonkeySafeLeaderTest, TestLMDirichletSimilarityFactory, TestSimpleTrackingShardHandler, AnalysisErrorHandlingTest, OpenExchangeRatesOrgProviderTest, TestSystemIdResolver, LeaderElectionTest, TestExactStatsCache, HighlighterMaxOffsetTest, SolrRequestParserTest, HdfsBasicDistributedZk2Test, CurrencyFieldXmlFileTest, TestMiniSolrCloudCluster, RAMDirectoryFactoryTest, TestCollectionAPI, PrimUtilsTest, HdfsCollectionsAPIDistributedZkTest, TestPostingsSolrHighlighter, TestSerializedLuceneMatchVersion, TestFieldTypeCollectionResource, TestJmxIntegration, TestPseudoReturnFields, TestSolrCoreProperties, TestImplicitCoreProperties, XsltUpdateRequestHandlerTest, CSVRequestHandlerTest, TestSolrJ, DateMathParserTest, TestTrie, TestRemoteStreaming, DistanceUnitsTest, TestEmbeddedSolrServerConstructors, CustomCollectionTest, ReplicationFactorTest]
   [junit4] Completed on J3 in 572.42s, 1 test, 1 error <<< FAILURES!

[...truncated 868 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1905 tests, 1 error, 61 ignored (21 assumptions)

Total time: 60 minutes 12 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 204630843 bytes
Compression is 0.0%
Took 36 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2906 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2906/

2 tests failed.
REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:35385//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:35385//collection1
	at __randomizedtesting.SeedInfo.seed([2902CDAD2A4D727B:A156F27784B11F83]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
java.net.SocketTimeoutException: Read timed out

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: java.net.SocketTimeoutException: Read timed out
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:774)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:277)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9961 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/init-core-data-001
   [junit4]   2> 677096 T2342 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 677103 T2342 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 677103 T2343 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 677203 T2342 oasc.ZkTestServer.run start zk server on port:18161
   [junit4]   2> 677204 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 677205 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 677210 T2350 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d07fe53 name:ZooKeeperConnection Watcher:127.0.0.1:18161 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 677210 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 677211 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 677211 T2342 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 677215 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 677215 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 677217 T2353 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5a5efbc7 name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 677217 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 677218 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 677218 T2342 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 677221 T2342 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 677222 T2342 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 677224 T2342 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 677226 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 677227 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 677229 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 677230 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 677332 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 677332 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 677334 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 677335 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 677336 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 677337 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 677338 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 677339 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 677340 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 677341 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 677342 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 677343 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 677344 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 677345 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 677346 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 677347 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 677348 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 677349 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 678800 T2342 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1
   [junit4]   2> 678803 T2342 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 678805 T2342 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:56011
   [junit4]   2> 678805 T2342 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=23759, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores}
   [junit4]   2> 678806 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 678806 T2342 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/'
   [junit4]   2> 678841 T2342 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/solr.xml
   [junit4]   2> 678883 T2342 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores
   [junit4]   2> 678884 T2342 oasc.CoreContainer.<init> New CoreContainer 2012355157
   [junit4]   2> 678884 T2342 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/]
   [junit4]   2> 678884 T2342 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/lib
   [junit4]   2> 678885 T2342 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/lib).
   [junit4]   2> 678895 T2342 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 678895 T2342 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 678895 T2342 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 678897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 678897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 678897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 678902 T2342 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 678903 T2342 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 678903 T2342 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 678903 T2342 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 678904 T2342 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18161/solr
   [junit4]   2> 678904 T2342 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 678904 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 678905 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 678906 T2365 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c251cb1 name:ZooKeeperConnection Watcher:127.0.0.1:18161 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 678907 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 678907 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 678909 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 678910 T2368 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69f080ec name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 678911 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 678912 T2342 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 678913 T2342 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 678915 T2342 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 678916 T2342 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 678917 T2342 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 678919 T2342 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 678920 T2342 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 678921 T2342 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 678922 T2342 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:23759_
   [junit4]   2> 678922 T2342 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:23759_
   [junit4]   2> 678923 T2342 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 678924 T2342 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 678925 T2342 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 678926 T2342 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:23759_
   [junit4]   2> 678926 T2342 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 678927 T2342 oasc.Overseer.start Overseer (id=93613545977020419-127.0.0.1:23759_-n_0000000000) starting
   [junit4]   2> 678929 T2342 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 678933 T2342 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 678933 T2342 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 678934 T2369 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 678933 T2370 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 678936 T2342 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores
   [junit4]   2> 678938 T2342 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, schema=schema.xml, collection=control_collection, transient=false, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1, shard=, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/, name=collection1, loadOnStartup=true}
   [junit4]   2> 678938 T2342 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/
   [junit4]   2> 678939 T2342 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 678939 T2372 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 678939 T2372 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 678940 T2372 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 678940 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 678941 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:23759_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 678941 T2369 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:23759_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 678941 T2369 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 678942 T2369 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 678942 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 679940 T2372 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 679941 T2372 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 679941 T2372 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 679942 T2372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 679942 T2372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 679942 T2372 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/'
   [junit4]   2> 679973 T2372 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 679976 T2372 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 679988 T2372 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 680014 T2372 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 680070 T2372 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 680172 T2372 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 680183 T2372 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 680625 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 680642 T2372 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 680646 T2372 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 680655 T2372 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 680660 T2372 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 680664 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 680665 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 680666 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 680666 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 680667 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 680667 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 680667 T2372 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 680667 T2372 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 680668 T2372 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data/
   [junit4]   2> 680668 T2372 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@542890fb
   [junit4]   2> 680669 T2372 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data
   [junit4]   2> 680669 T2372 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data/index/
   [junit4]   2> 680669 T2372 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 680670 T2372 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data/index
   [junit4]   2> 680670 T2372 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=13, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 680671 T2372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@a0b4a2f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b3358cf),segFN=segments_1,generation=1}
   [junit4]   2> 680671 T2372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 680678 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 680678 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 680679 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 680679 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 680679 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 680679 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 680680 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 680680 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 680680 T2372 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 680700 T2372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 680703 T2372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 680705 T2372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 680708 T2372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 680713 T2372 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/admin/system,/update/json,/update,/update/json/docs,/admin/segments,/admin/luke,/admin/file,/schema,/get,/admin/mbeans,standard,/admin/threads,/admin/plugins,/admin/properties,/update/csv,/admin/ping,/config,/admin/logging
   [junit4]   2> 680715 T2372 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 680717 T2372 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 680717 T2372 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 680718 T2372 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 680719 T2372 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 680720 T2372 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1609071471, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 680721 T2372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@a0b4a2f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b3358cf),segFN=segments_1,generation=1}
   [junit4]   2> 680721 T2372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 680721 T2372 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4ab3b662[collection1] main
   [junit4]   2> 680722 T2372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 680722 T2372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 680723 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 680723 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 680723 T2372 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 680723 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 680724 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 680724 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 680724 T2372 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 680725 T2372 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 680725 T2373 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4ab3b662[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 680726 T2372 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 680727 T2376 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:23759 collection:control_collection shard:shard1
   [junit4]   2> 680727 T2342 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 680727 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 680727 T2376 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 680729 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 680730 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 680731 T2380 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@471e5cb name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 680731 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 680732 T2376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 680732 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 680732 T2342 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 680733 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 680733 T2376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 680733 T2376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C556 name=collection1 org.apache.solr.core.SolrCore@e402c82 url=http://127.0.0.1:23759/collection1 node=127.0.0.1:23759_ C556_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:23759_, state=down, base_url=http://127.0.0.1:23759, core=collection1}
   [junit4]   2> 680734 T2376 C:control_collection S:shard1 C556 P23759 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:23759/collection1/
   [junit4]   2> 680734 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 680734 T2376 C:control_collection S:shard1 C556 P23759 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 680734 T2376 C:control_collection S:shard1 C556 P23759 oasc.SyncStrategy.syncToMe http://127.0.0.1:23759/collection1/ has no replicas
   [junit4]   2> 680734 T2376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:23759/collection1/ shard1
   [junit4]   2> 680734 T2342 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 680735 T2376 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 680735 T2342 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 680735 T2380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680735 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 680735 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680736 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 680738 T2383 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@57707db6 name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 680738 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 680738 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 680740 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 680741 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 680742 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 680743 T2369 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 680743 T2380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680743 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680746 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 680746 T2369 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 680747 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680747 T2380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680790 T2376 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:23759/collection1/ and leader is http://127.0.0.1:23759/collection1/
   [junit4]   2> 680791 T2376 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:23759
   [junit4]   2> 680791 T2376 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 680791 T2376 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 680791 T2376 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 680793 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 680794 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:23759_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 4
   [junit4]   2> 680794 T2369 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:23759_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 680898 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680898 T2380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 681807 T2342 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1
   [junit4]   2> 681810 T2342 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001
   [junit4]   2> 681810 T2342 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 681813 T2342 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:63035
   [junit4]   2> 681813 T2342 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=15820, solrconfig=solrconfig.xml, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores}
   [junit4]   2> 681814 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 681814 T2342 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/'
   [junit4]   2> 681847 T2342 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/solr.xml
   [junit4]   2> 681886 T2342 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores
   [junit4]   2> 681887 T2342 oasc.CoreContainer.<init> New CoreContainer 915242244
   [junit4]   2> 681887 T2342 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/]
   [junit4]   2> 681887 T2342 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/lib
   [junit4]   2> 681888 T2342 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/lib).
   [junit4]   2> 681897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 681897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 681897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 681899 T2342 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 681899 T2342 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 681899 T2342 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 681904 T2342 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 681905 T2342 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 681905 T2342 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 681905 T2342 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 681905 T2342 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18161/solr
   [junit4]   2> 681906 T2342 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 681906 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 681907 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 681908 T2395 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@510500e8 name:ZooKeeperConnection Watcher:127.0.0.1:18161 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 681909 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 681910 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 681912 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 681913 T2398 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f1aea5 name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 681914 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 681918 T2342 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 682922 T2342 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:15820_
   [junit4]   2> 682923 T2342 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:15820_
   [junit4]   2> 682926 T2342 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 682929 T2342 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores
   [junit4]   2> 682931 T2342 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, loadOnStartup=true, dataDir=data/, name=collection1, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/, shard=, schema=schema.xml, collection=collection1, transient=false}
   [junit4]   2> 682932 T2342 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/
   [junit4]   2> 682932 T2342 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 682934 T2399 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 682934 T2399 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 682935 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 682936 T2399 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 682936 T2399 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 682936 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:15820_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 682936 T2399 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 682937 T2369 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:15820_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 682937 T2369 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 682937 T2369 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 682937 T2399 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 682937 T2399 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 682938 T2369 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 682939 T2398 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 682939 T2398 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 683938 T2399 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 683939 T2399 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 683939 T2399 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 683939 T2399 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 683940 T2399 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 683940 T2399 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/'
   [junit4]   2> 683953 T2399 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 683955 T2399 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 683962 T2399 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 683977 T2399 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 684010 T2399 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 684111 T2399 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 684122 T2399 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 684548 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 684564 T2399 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 684567 T2399 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 684577 T2399 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 684582 T2399 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 684586 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 684587 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 684587 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 684587 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 684588 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 684588 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 684588 T2399 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 684588 T2399 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 684589 T2399 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 684589 T2399 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@542890fb
   [junit4]   2> 684590 T2399 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data
   [junit4]   2> 684590 T2399 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 684591 T2399 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 684591 T2399 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 684591 T2399 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=13, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 684592 T2399 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2758c73a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@357fc504),segFN=segments_1,generation=1}
   [junit4]   2> 684593 T2399 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 684600 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 684600 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 684600 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 684601 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 684601 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 684601 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 684602 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 684602 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 684602 T2399 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 684621 T2399 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 684624 T2399 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 684626 T2399 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 684628 T2399 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 684635 T2399 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/admin/system,/update/json,/update,/update/json/docs,/admin/segments,/admin/luke,/admin/file,/schema,/get,/admin/mbeans,standard,/admin/threads,/admin/plugins,/admin/properties,/update/csv,/admin/ping,/config,/admin/logging
   [junit4]   2> 684637 T2399 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 684639 T2399 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 684639 T2399 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 684641 T2399 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 684641 T2399 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 684642 T2399 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1609071471, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 684643 T2399 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2758c73a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@357fc504),segFN=segments_1,generation=1}
   [junit4]   2> 684643 T2399 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 684643 T2399 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@ebe3283[collection1] main
   [junit4]   2> 684644 T2399 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 684644 T2399 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 684645 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 684645 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 684645 T2399 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 684645 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 684646 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 684646 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 684646 T2399 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 684647 T2399 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 684648 T2400 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@ebe3283[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 684648 T2399 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 684649 T2403 C:collection1 S:shard2 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:15820 collection:collection1 shard:shard2
   [junit4]   2> 684649 T2342 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 684650 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 684650 T2403 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 684654 T2403 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 684656 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 684656 T2403 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 684656 T2403 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C557 name=collection1 org.apache.solr.core.SolrCore@6815492e url=http://127.0.0.1:15820/collection1 node=127.0.0.1:15820_ C557_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:15820_, state=down, base_url=http://127.0.0.1:15820, core=collection1}
   [junit4]   2> 684656 T2403 C:collection1 S:shard2 C557 P15820 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:15820/collection1/
   [junit4]   2> 684656 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 684656 T2403 C:collection1 S:shard2 C557 P15820 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 684657 T2403 C:collection1 S:shard2 C557 P15820 oasc.SyncStrategy.syncToMe http://127.0.0.1:15820/collection1/ has no replicas
   [junit4]   2> 684657 T2403 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:15820/collection1/ shard2
   [junit4]   2> 684657 T2403 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 684657 T2369 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 684658 T2398 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 684659 T2398 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 684661 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 684661 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 684662 T2369 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 684663 T2398 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 684663 T2398 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 684712 T2403 C:collection1 S:shard2 oasc.ZkController.register We are http://127.0.0.1:15820/collection1/ and leader is http://127.0.0.1:15820/collection1/
   [junit4]   2> 684713 T2403 C:collection1 S:shard2 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:15820
   [junit4]   2> 684713 T2403 C:collection1 S:shard2 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 684714 T2403 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 684714 T2403 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 684715 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 684716 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:15820_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 684717 T2369 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:15820_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 684719 T2369 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 684719 T2403 C:collection1 S:shard2 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 684720 T2398 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 684720 T2398 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 686203 T2342 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/cores/collection1
   [junit4]   2> 686206 T2342 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001
   [junit4]   2> 686206 T2342 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 686209 T2342 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:26237
   [junit4]   2> 686209 T2342 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=26236, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/tempDir-001/jetty2, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/cores, solrconfig=solrconfig.xml}
   [junit4]   2> 686210 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 686210 T2342 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/'
   [junit4]   2> 686243 T2342 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/solr.xml
   [junit4]   2> 686284 T2342 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/cores
   [junit4]   2> 686285 T2342 oasc.CoreContainer.<init> New CoreContainer 1800781633
   [junit4]   2> 686285 T2342 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/]
   [junit4]   2> 686285 T2342 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/lib
   [junit4]   2> 686286 T2342 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/lib).
   [junit4]   2> 686295 T2342 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 686296 T2342 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 686296 T2342 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 686296 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 686296 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 686297 T2342 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 686297 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 686297 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 686297 T2342 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 686298 T2342 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 686298 T2342 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 686303 T2342 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 686304 T2342 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 686304 T2342 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFa

[...truncated too long message...]

lr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 2902CDAD2A4D727B-001/tempDir-001/shard2/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 2902CDAD2A4D727B-001/tempDir-001/shard2/cores/collection1/data;done=false>>]
   [junit4]   2> 2026438 T7432 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 2902CDAD2A4D727B-001/tempDir-001/shard2/cores/collection1/data
   [junit4]   2> 2026439 T7432 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedMLTComponentTest -Dtests.method=test -Dtests.seed=2902CDAD2A4D727B -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ru -Dtests.timezone=Asia/Chungking -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    104s J0 | DistributedMLTComponentTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:35385//collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2902CDAD2A4D727B:A156F27784B11F83]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
   [junit4]    > 	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:152)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:122)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 51 more
   [junit4]   2> 2026642 T7432 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 2902CDAD2A4D727B-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {lowerfilt=PostingsFormat(name=MockRandom), intDefault=Lucene50(blocksize=128), lowerfilt1=PostingsFormat(name=Memory doPackFST= false), multiDefault=PostingsFormat(name=MockRandom), range_facet_l=PostingsFormat(name=MockRandom), timestamp=PostingsFormat(name=MockRandom), id=FSTOrd50, _version_=Lucene50(blocksize=128), lowerfilt1and2=PostingsFormat(name=MockRandom)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=ru, timezone=Asia/Chungking
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=229808104,total=371720192
   [junit4]   2> NOTE: All tests run in this JVM: [TestRecoveryHdfs, SpellCheckComponentTest, SynonymTokenizerTest, OverseerRolesTest, SpatialHeatmapFacetsTest, TestChildDocTransformer, TestSchemaVersionResource, XmlUpdateRequestHandlerTest, BJQParserTest, TestBM25SimilarityFactory, TestExtendedDismaxParser, AddSchemaFieldsUpdateProcessorFactoryTest, TestOrdValues, ClusterStateUpdateTest, TestFastOutputStream, DeleteShardTest, TestHdfsUpdateLog, UpdateRequestProcessorFactoryTest, DirectSolrSpellCheckerTest, CoreMergeIndexesAdminHandlerTest, CoreAdminRequestStatusTest, TestLMDirichletSimilarityFactory, HdfsUnloadDistributedZkTest, SystemInfoHandlerTest, TestSimpleTrackingShardHandler, CloudMLTQParserTest, DistanceUnitsTest, TestReqParamsAPI, DisMaxRequestHandlerTest, AtomicUpdatesTest, TestDocBasedVersionConstraints, QueryResultKeyTest, TestUniqueKeyFieldResource, TestAnalyzedSuggestions, SpellCheckCollatorTest, TestStressLucene, TestCodecSupport, SchemaVersionSpecificBehaviorTest, TermsComponentTest, OverriddenZkACLAndCredentialsProvidersTest, DistributedSuggestComponentTest, SolrInfoMBeanTest, TestFieldSortValues, AddBlockUpdateTest, TestReplicaProperties, TestBlobHandler, TestFuzzyAnalyzedSuggestions, OutputWriterTest, BadCopyFieldTest, TestBinaryResponseWriter, DistributedFacetPivotSmallAdvancedTest, TestDefaultSearchFieldResource, CachingDirectoryFactoryTest, FileBasedSpellCheckerTest, TestJsonFacets, DOMUtilTest, RollingRestartTest, HttpPartitionTest, ShardSplitTest, TestCloudSchemaless, FullSolrCloudDistribCmdsTest, TestSolrConfigHandlerConcurrent, TestDynamicLoading, CollectionReloadTest, TestRebalanceLeaders, CloudExitableDirectoryReaderTest, OverseerStatusTest, TestDistribDocBasedVersion, TestShortCircuitedRequests, DistributedFacetPivotLargeTest, ZkSolrClientTest, TestHashPartitioner, TestReload, ZkCLITest, DistributedMLTComponentTest]
   [junit4] Completed on J0 in 103.81s, 1 test, 1 error <<< FAILURES!

[...truncated 413 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 2 errors, 61 ignored (21 assumptions)

Total time: 58 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 212829115 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2905 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2905/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:21000/p_se/a/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21000/p_se/a/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([DFC12658F5827783:579519825B7E1A7B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:21000 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28440/_xv/b/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28440/_xv/b/collection1
	at __randomizedtesting.SeedInfo.seed([DFC12658F5827783:579519825B7E1A7B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28440/_xv/b/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:28440 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22580/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22580/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([DFC12658F5827783:579519825B7E1A7B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22580/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:22580 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9544 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/init-core-data-001
   [junit4]   2> 228752 T610 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 228762 T610 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 228763 T611 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 228862 T610 oasc.ZkTestServer.run start zk server on port:22563
   [junit4]   2> 228863 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 228864 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 228873 T618 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@37715099 name:ZooKeeperConnection Watcher:127.0.0.1:22563 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 228873 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 228874 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 228874 T610 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 228878 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 228879 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 228882 T621 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@42f758cf name:ZooKeeperConnection Watcher:127.0.0.1:22563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 228882 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 228883 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 228883 T610 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 228886 T610 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 228888 T610 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 228890 T610 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 228893 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 228895 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 228900 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 228901 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 228905 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 228906 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 228909 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 228911 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 228915 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 228916 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 228920 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 228922 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 228927 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 228929 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 228933 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 228935 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 228939 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 228940 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 228947 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 228949 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 228954 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 228955 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 233138 T610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1
   [junit4]   2> 233142 T610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 233145 T610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22581
   [junit4]   2> 233145 T610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/tempDir-001/control/data, hostContext=/, hostPort=22580, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores}
   [junit4]   2> 233146 T610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 233146 T610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/'
   [junit4]   2> 233194 T610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/solr.xml
   [junit4]   2> 233407 T610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores
   [junit4]   2> 233408 T610 oasc.CoreContainer.<init> New CoreContainer 957384846
   [junit4]   2> 233408 T610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/]
   [junit4]   2> 233409 T610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/lib
   [junit4]   2> 233413 T610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/lib).
   [junit4]   2> 233446 T610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 233446 T610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 233447 T610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 233447 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 233447 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 233448 T610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 233448 T610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 233449 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 233450 T610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 233450 T610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 233450 T610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 233475 T610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 233478 T610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 233478 T610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 233478 T610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 233479 T610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22563/solr
   [junit4]   2> 233479 T610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 233480 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 233482 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 233484 T633 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2258256e name:ZooKeeperConnection Watcher:127.0.0.1:22563 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 233485 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 233486 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 233494 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 233496 T636 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6b2e0f34 name:ZooKeeperConnection Watcher:127.0.0.1:22563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 233496 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 233499 T610 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 233506 T610 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 233512 T610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 233517 T610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 233521 T610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 233534 T610 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 233542 T610 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 233547 T610 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 233552 T610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22580_
   [junit4]   2> 233553 T610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22580_
   [junit4]   2> 233557 T610 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 233560 T610 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 233565 T610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 233569 T610 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:22580_
   [junit4]   2> 233569 T610 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 233574 T610 oasc.Overseer.start Overseer (id=93612242919948291-127.0.0.1:22580_-n_0000000000) starting
   [junit4]   2> 233579 T610 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 233594 T610 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 233595 T638 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 233595 T610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 233596 T637 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 233603 T610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores
   [junit4]   2> 233611 T610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, coreNodeName=, schema=schema.xml, transient=false, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/, shard=, config=solrconfig.xml, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1, name=collection1}
   [junit4]   2> 233612 T610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/
   [junit4]   2> 233612 T610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 233614 T640 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 233614 T640 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 233616 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 233616 T640 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 233618 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22580_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 233620 T637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22580_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 233620 T637 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 233620 T637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 233624 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 234617 T640 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 234617 T640 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 234619 T640 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 234620 T640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 234621 T640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 234621 T640 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/'
   [junit4]   2> 234651 T640 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 234658 T640 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 234686 T640 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 234753 T640 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 234888 T640 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 234890 T640 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 234913 T640 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 235913 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 235952 T640 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 235957 T640 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 235977 T640 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 235986 T640 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 235993 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 235995 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 235996 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 235997 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 235998 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 235998 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 235999 T640 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 235999 T640 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 236000 T640 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data/
   [junit4]   2> 236000 T640 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f14be2e
   [junit4]   2> 236001 T640 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data
   [junit4]   2> 236002 T640 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data/index/
   [junit4]   2> 236002 T640 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 236003 T640 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data/index
   [junit4]   2> 236004 T640 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6812259126929583]
   [junit4]   2> 236005 T640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7f765e23 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@341e8d35),segFN=segments_1,generation=1}
   [junit4]   2> 236005 T640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 236014 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 236014 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 236015 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 236015 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 236015 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 236016 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 236017 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 236017 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 236017 T640 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 236039 T640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 236041 T640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 236043 T640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 236045 T640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 236049 T640 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/admin/system,/admin/file,/admin/logging,/admin/luke,standard,/update/csv,/admin/ping,/update/json,/admin/threads,/admin/mbeans,/admin/properties,/config,/get,/admin/segments,/update,/replication,/update/json/docs,/admin/plugins
   [junit4]   2> 236051 T640 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 236053 T640 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 236053 T640 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 236055 T640 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 236055 T640 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 236055 T640 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=50.4541015625, floorSegmentMB=0.912109375, forceMergeDeletesPctAllowed=9.525263553538434, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.16973219653750882
   [junit4]   2> 236056 T640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7f765e23 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@341e8d35),segFN=segments_1,generation=1}
   [junit4]   2> 236056 T640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 236057 T640 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6b9df4b3[collection1] main
   [junit4]   2> 236057 T640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 236058 T640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 236058 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 236059 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 236059 T640 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 236059 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 236060 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 236060 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 236060 T640 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 236061 T640 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 236061 T641 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6b9df4b3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 236062 T640 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 236063 T644 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22580 collection:control_collection shard:shard1
   [junit4]   2> 236063 T610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 236063 T610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 236063 T644 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 236067 T644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 236068 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 236068 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 236068 T644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 236068 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 236069 T644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C67 name=collection1 org.apache.solr.core.SolrCore@347de428 url=http://127.0.0.1:22580/collection1 node=127.0.0.1:22580_ C67_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:22580, core=collection1, node_name=127.0.0.1:22580_}
   [junit4]   2> 236069 T644 C:control_collection S:shard1 C67 P22580 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22580/collection1/
   [junit4]   2> 236070 T644 C:control_collection S:shard1 C67 P22580 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 236070 T644 C:control_collection S:shard1 C67 P22580 oasc.SyncStrategy.syncToMe http://127.0.0.1:22580/collection1/ has no replicas
   [junit4]   2> 236070 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 236070 T648 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b4907d8 name:ZooKeeperConnection Watcher:127.0.0.1:22563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 236070 T644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22580/collection1/ shard1
   [junit4]   2> 236070 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 236071 T644 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 236071 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 236071 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 236072 T610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 236074 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 236074 T610 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 236075 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 236178 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 236178 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 236227 T644 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:22580/collection1/ and leader is http://127.0.0.1:22580/collection1/
   [junit4]   2> 236228 T644 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22580
   [junit4]   2> 236228 T644 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 236228 T644 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 236228 T644 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 236230 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 236231 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22580_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 236231 T637 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22580_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 236334 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 236334 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 239237 T610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1
   [junit4]   2> 239249 T610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001
   [junit4]   2> 239251 T610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 239259 T610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22602
   [junit4]   2> 239259 T610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/tempDir-001/jetty1, hostContext=/, solrconfig=solrconfig.xml, hostPort=22601}
   [junit4]   2> 239260 T610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 239261 T610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/'
   [junit4]   2> 239405 T610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/solr.xml
   [junit4]   2> 239472 T610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores
   [junit4]   2> 239472 T610 oasc.CoreContainer.<init> New CoreContainer 1684710647
   [junit4]   2> 239473 T610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/]
   [junit4]   2> 239473 T610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/lib
   [junit4]   2> 239474 T610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/lib).
   [junit4]   2> 239487 T610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 239488 T610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 239488 T610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 239489 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 239489 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 239489 T610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 239489 T610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 239490 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 239490 T610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 239490 T610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 239491 T610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 239497 T610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 239498 T610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 239498 T610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 239499 T610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 239499 T610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22563/solr
   [junit4]   2> 239500 T610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 239500 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 239501 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 239504 T660 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74ed31e5 name:ZooKeeperConnection Watcher:127.0.0.1:22563 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 239504 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 239505 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 239509 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 239512 T663 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4faae9f3 name:ZooKeeperConnection Watcher:127.0.0.1:22563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 239512 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 239518 T610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 240529 T610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22601_
   [junit4]   2> 240531 T610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22601_
   [junit4]   2> 240543 T610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 240551 T610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores
   [junit4]   2> 240560 T610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/, collection=collection1, name=collection1, shard=, loadOnStartup=true, schema=schema.xml, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1, config=solrconfig.xml, dataDir=data/}
   [junit4]   2> 240561 T610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/
   [junit4]   2> 240562 T610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 240563 T664 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 240564 T664 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 240566 T664 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 240566 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 240567 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:22601_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 240568 T637 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:22601_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 240568 T637 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 240569 T637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 240572 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 240572 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 240573 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 241566 T664 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 241567 T664 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 241568 T664 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 241569 T664 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 241569 T664 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 241570 T664 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/'
   [junit4]   2> 241604 T664 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 241615 T664 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 241641 T664 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 241698 T664 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 241785 T664 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 241787 T664 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 241807 T664 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 242758 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 242775 T664 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 242778 T664 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 242812 T664 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 242817 T664 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 242821 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 242822 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 242822 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 242823 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 242823 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 242823 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 242824 T664 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 242824 T664 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 242824 T664 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 242825 T664 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f14be2e
   [junit4]   2> 242825 T664 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data
   [junit4]   2> 242826 T664 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 242826 T664 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 242826 T664 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 242827 T664 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6812259126929583]
   [junit4]   2> 242828 T664 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3d1ff8dc lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7e0b52dd),segFN=segments_1,generation=1}
   [junit4]   2> 242828 T664 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 242834 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 242834 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 242835 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 242835 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 242835 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 242835 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 242836 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 242836 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 242836 T664 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 242853 T664 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 242855 T664 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 242857 T664 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 242859 T664 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 242867 T664 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/admin/system,/admin/file,/admin/logging,/admin/luke,standard,/update/csv,/admin/ping,/update/json,/admin/threads,/admin/mbeans,/admin/properties,/config,/get,/admin/segments,/update,/replication,/update/json/docs,/admin/plugins
   [junit4]   2> 242869 T664 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 242871 T664 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 242871 T664 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 242873 T664 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 242873 T664 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 242874 T664 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=50.4541015625, floorSegmentMB=0.912109375, forceMergeDeletesPctAllowed=9.525263553538434, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.16973219653750882
   [junit4]   2> 242876 T664 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3d1ff8dc lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7e0b52dd),segFN=segments_1,generation=1}
   [junit4]   2> 242876 T664 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 242876 T664 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@63471445[collection1] main
   [junit4]   2> 242877 T664 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 242880 T664 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 242880 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 242884 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 242884 T664 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 242884 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 242885 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 242885 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 242886 T664 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 242886 T664 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 242888 T665 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@63471445[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 242889 T664 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 242890 T668 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22601 collection:collection1 shard:shard3
   [junit4]   2> 242891 T610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 242892 T610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 242892 T668 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 242905 T668 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 242908 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 242909 T668 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 242909 T668 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C68 name=collection1 org.apache.solr.core.SolrCore@65a55801 url=http://127.0.0.1:22601/collection1 node=127.0.0.1:22601_ C68_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:22601, core=collection1, node_name=127.0.0.1:22601_}
   [junit4]   2> 242910 T668 C:collection1 S:shard3 C68 P22601 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22601/collection1/
   [junit4]   2> 242910 T668 C:collection1 S:shard3 C68 P22601 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 242910 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 242911 T668 C:collection1 S:shard3 C68 P22601 oasc.SyncStrategy.syncToMe http://127.0.0.1:22601/collection1/ has no replicas
   [junit4]   2> 242912 T668 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22601/collection1/ shard3
   [junit4]   2> 242912 T668 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 242918 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 242919 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 242918 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 242924 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 242926 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 243035 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243035 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243035 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243079 T668 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:22601/collection1/ and leader is http://127.0.0.1:22601/collection1/
   [junit4]   2> 243079 T668 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22601
   [junit4]   2> 243080 T668 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 243080 T668 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 243080 T668 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 243083 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 243085 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22601_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 7
   [junit4]   2> 243086 T637 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22601_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 243194 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243195 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243194 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 247015 T610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-2-001/cores/collection1
   [junit4]   2> 247025 T610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/sol

[...truncated too long message...]

etContextHandler{/p_se/a,null}
   [junit4]   2> 1487445 T2508 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:20927 20927
   [junit4]   2> 1487791 T2509 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:20927 20927
   [junit4]   2> 1487793 T2509 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1487793 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21008/p_se/a
   [junit4]   2> 1487793 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:20963/p_se/a
   [junit4]   2> 1487794 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20980/p_se/a
   [junit4]   2> 1487794 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:21000/p_se/a
   [junit4]   2> 1487794 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20951/p_se/a
   [junit4]   2> 1487795 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21008/p_se/a
   [junit4]   2> 1487795 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20963/p_se/a
   [junit4]   2> 1487795 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20980/p_se/a
   [junit4]   2> 1487795 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21000/p_se/a
   [junit4]   2> 1487796 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20951/p_se/a
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=DFC12658F5827783 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=en_PH -Dtests.timezone=America/Paramaribo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   92.8s J0 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21000/p_se/a/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DFC12658F5827783:579519825B7E1A7B]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:21000 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1487804 T2508 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DFC12658F5827783-001
   [junit4]   2> 92813 T2507 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=en_PH, timezone=America/Paramaribo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=85241336,total=286261248
   [junit4]   2> NOTE: All tests run in this JVM: [SolrInfoMBeanTest, JsonLoaderTest, TestJmxIntegration, ShardRoutingTest, ExitableDirectoryReaderTest, CSVRequestHandlerTest, TestSweetSpotSimilarityFactory, TestSuggestSpellingConverter, HighlighterMaxOffsetTest, OverseerRolesTest, TestLuceneMatchVersion, ReplicationFactorTest, TestShortCircuitedRequests, DateRangeFieldTest, DocumentBuilderTest, CacheHeaderTest, RecoveryZkTest, TestInitParams, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestJettySolrRunner, TestSerializedLuceneMatchVersion, TestSchemaResource, TestTrackingShardHandlerFactory, PathHierarchyTokenizerFactoryTest, TestDynamicFieldResource, BasicFunctionalityTest, RequiredFieldsTest, SimpleCollectionCreateDeleteTest, TestExactSharedStatsCache, TestFileDictionaryLookup, TestFunctionQuery, TestClusterStateMutator, TestManagedResourceStorage, AnalysisErrorHandlingTest, RemoteQueryErrorTest, TestSolrDeletionPolicy2, HttpPartitionTest, AnalyticsQueryTest, TestTrie, XsltUpdateRequestHandlerTest, TestConfigOverlay, TestInitQParser, ScriptEngineTest, TestCollapseQParserPlugin, DirectUpdateHandlerTest, TestQueryTypes, TestRecovery, SolrIndexSplitterTest, AliasIntegrationTest, TestReloadAndDeleteDocs, HdfsDirectoryTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J0 in 93.60s, 1 test, 1 error <<< FAILURES!

[...truncated 748 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 61 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 213820630 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2904 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2904/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:32152/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32152/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([998831C6A7E2310A:11DC0E1C091E5CF2]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:32152 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37941/cpwc/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37941/cpwc/collection1
	at __randomizedtesting.SeedInfo.seed([998831C6A7E2310A:11DC0E1C091E5CF2]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37941/cpwc/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:37941 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([998831C6A7E2310A:11DC0E1C091E5CF2]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:31600 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9355 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/init-core-data-001
   [junit4]   2> 160347 T395 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 160352 T395 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 160353 T396 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 160453 T395 oasc.ZkTestServer.run start zk server on port:32104
   [junit4]   2> 160454 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 160455 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 160458 T403 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c3d4223 name:ZooKeeperConnection Watcher:127.0.0.1:32104 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 160458 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 160459 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 160459 T395 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 160462 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 160462 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 160463 T406 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c4b078f name:ZooKeeperConnection Watcher:127.0.0.1:32104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 160464 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 160464 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 160464 T395 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 160466 T395 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 160468 T395 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 160469 T395 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 160471 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 160472 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 160475 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 160475 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 160478 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 160478 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 160480 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 160480 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 160483 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 160483 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 160485 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 160486 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 160488 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 160489 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 160490 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 160491 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 160492 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 160493 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 160495 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 160495 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 160497 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 160497 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 161891 T395 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1
   [junit4]   2> 161896 T395 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 161900 T395 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32111
   [junit4]   2> 161900 T395 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=32110, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/tempDir-001/control/data}
   [junit4]   2> 161901 T395 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 161901 T395 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/'
   [junit4]   2> 161953 T395 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/solr.xml
   [junit4]   2> 162016 T395 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores
   [junit4]   2> 162017 T395 oasc.CoreContainer.<init> New CoreContainer 139305003
   [junit4]   2> 162017 T395 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/]
   [junit4]   2> 162018 T395 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/lib
   [junit4]   2> 162019 T395 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/lib).
   [junit4]   2> 162033 T395 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 162033 T395 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 162033 T395 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 162034 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 162034 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 162034 T395 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 162035 T395 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 162035 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 162035 T395 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 162035 T395 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 162036 T395 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 162043 T395 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 162045 T395 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 162045 T395 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 162045 T395 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 162046 T395 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32104/solr
   [junit4]   2> 162046 T395 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 162046 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 162047 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 162049 T418 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a84a043 name:ZooKeeperConnection Watcher:127.0.0.1:32104 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 162049 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 162050 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 162052 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 162054 T421 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22497601 name:ZooKeeperConnection Watcher:127.0.0.1:32104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 162054 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 162056 T395 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 162059 T395 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 162061 T395 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 162063 T395 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 162066 T395 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 162068 T395 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 162070 T395 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 162072 T395 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 162073 T395 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32110_
   [junit4]   2> 162074 T395 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32110_
   [junit4]   2> 162076 T395 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 162077 T395 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 162079 T395 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 162081 T395 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:32110_
   [junit4]   2> 162081 T395 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 162083 T395 oasc.Overseer.start Overseer (id=93611275814240259-127.0.0.1:32110_-n_0000000000) starting
   [junit4]   2> 162085 T395 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 162092 T395 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 162093 T423 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 162093 T395 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 162093 T422 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 162098 T395 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores
   [junit4]   2> 162101 T395 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/, shard=, name=collection1, schema=schema.xml, config=solrconfig.xml, loadOnStartup=true, transient=false, collection=control_collection, coreNodeName=, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1}
   [junit4]   2> 162101 T395 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/
   [junit4]   2> 162101 T395 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 162103 T425 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 162103 T425 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 162104 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 162104 T425 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 162105 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "node_name":"127.0.0.1:32110_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 162106 T422 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "node_name":"127.0.0.1:32110_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 162106 T422 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 162106 T422 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 162108 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 163105 T425 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 163106 T425 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 163107 T425 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 163107 T425 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 163108 T425 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 163108 T425 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/'
   [junit4]   2> 163127 T425 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 163130 T425 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 163145 T425 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 163175 T425 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 163278 T425 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 163280 T425 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 163297 T425 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 163690 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 163703 T425 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 163706 T425 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 163722 T425 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 163727 T425 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 163730 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 163731 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 163732 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 163732 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 163733 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 163733 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 163733 T425 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 163733 T425 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 163734 T425 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data/
   [junit4]   2> 163734 T425 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f3fd958
   [junit4]   2> 163734 T425 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data
   [junit4]   2> 163735 T425 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data/index/
   [junit4]   2> 163735 T425 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 163735 T425 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data/index
   [junit4]   2> 163736 T425 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=73840210, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 163736 T425 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@270b6d1c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5f285d4a),segFN=segments_1,generation=1}
   [junit4]   2> 163736 T425 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 163741 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 163741 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 163741 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 163743 T425 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 163757 T425 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 163759 T425 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 163760 T425 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 163762 T425 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 163766 T425 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/replication,/get,/admin/mbeans,/admin/properties,/update/csv,/update/json,/admin/logging,/admin/system,/admin/threads,/update,/admin/ping,/schema,/admin/plugins,/update/json/docs,/admin/segments,/config,standard,/admin/file
   [junit4]   2> 163768 T425 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 163769 T425 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 163769 T425 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 163770 T425 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 163771 T425 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 163771 T425 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=38, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10602606676023907]
   [junit4]   2> 163771 T425 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@270b6d1c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5f285d4a),segFN=segments_1,generation=1}
   [junit4]   2> 163772 T425 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 163772 T425 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7a00e70c[collection1] main
   [junit4]   2> 163772 T425 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 163773 T425 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 163773 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 163774 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 163774 T425 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 163774 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 163775 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 163775 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 163775 T425 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 163775 T425 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 163776 T426 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a00e70c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 163777 T425 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 163778 T429 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32110 collection:control_collection shard:shard1
   [junit4]   2> 163778 T395 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 163779 T395 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 163779 T429 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 163784 T429 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 163785 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 163785 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 163786 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 163787 T429 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 163787 T433 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@425a03e7 name:ZooKeeperConnection Watcher:127.0.0.1:32104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 163787 T429 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 163787 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 163787 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C107 name=collection1 org.apache.solr.core.SolrCore@7125ade5 url=http://127.0.0.1:32110/collection1 node=127.0.0.1:32110_ C107_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:32110, node_name=127.0.0.1:32110_, state=down}
   [junit4]   2> 163787 T429 C:control_collection S:shard1 C107 P32110 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32110/collection1/
   [junit4]   2> 163788 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 163788 T429 C:control_collection S:shard1 C107 P32110 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 163789 T395 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 163789 T429 C:control_collection S:shard1 C107 P32110 oasc.SyncStrategy.syncToMe http://127.0.0.1:32110/collection1/ has no replicas
   [junit4]   2> 163789 T429 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32110/collection1/ shard1
   [junit4]   2> 163789 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 163790 T429 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 163792 T395 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 163795 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 163796 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 163903 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 163903 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 163947 T429 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:32110/collection1/ and leader is http://127.0.0.1:32110/collection1/
   [junit4]   2> 163948 T429 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32110
   [junit4]   2> 163948 T429 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 163948 T429 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 163948 T429 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 163950 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 163951 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32110_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 163952 T422 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32110_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 164055 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 164055 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 166294 T395 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1
   [junit4]   2> 166298 T395 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001
   [junit4]   2> 166299 T395 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 166302 T395 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32132
   [junit4]   2> 166303 T395 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores, hostPort=32131, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/tempDir-001/jetty1, solrconfig=solrconfig.xml}
   [junit4]   2> 166304 T395 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 166305 T395 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/'
   [junit4]   2> 166365 T395 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/solr.xml
   [junit4]   2> 166431 T395 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores
   [junit4]   2> 166431 T395 oasc.CoreContainer.<init> New CoreContainer 1214137018
   [junit4]   2> 166431 T395 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/]
   [junit4]   2> 166432 T395 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/lib
   [junit4]   2> 166433 T395 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/lib).
   [junit4]   2> 166448 T395 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 166449 T395 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 166449 T395 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 166449 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 166450 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 166450 T395 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 166450 T395 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 166451 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 166451 T395 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 166451 T395 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 166452 T395 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 166460 T395 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 166462 T395 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 166462 T395 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 166462 T395 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 166463 T395 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32104/solr
   [junit4]   2> 166463 T395 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 166463 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 166464 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 166467 T445 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@48885506 name:ZooKeeperConnection Watcher:127.0.0.1:32104 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 166467 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 166468 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 166471 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 166473 T448 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b27c214 name:ZooKeeperConnection Watcher:127.0.0.1:32104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 166474 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 166482 T395 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 167488 T395 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32131_
   [junit4]   2> 167489 T395 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32131_
   [junit4]   2> 167491 T395 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 167494 T395 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores
   [junit4]   2> 167496 T395 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, name=collection1, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1, schema=schema.xml, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/, collection=collection1, dataDir=data/, coreNodeName=, shard=}
   [junit4]   2> 167497 T395 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/
   [junit4]   2> 167497 T395 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 167498 T449 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 167498 T449 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 167499 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 167499 T449 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 167500 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "node_name":"127.0.0.1:32131_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 167500 T422 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "node_name":"127.0.0.1:32131_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 167500 T422 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 167501 T422 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 167502 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 167502 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 167502 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 168499 T449 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 168500 T449 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 168501 T449 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 168501 T449 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 168502 T449 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 168502 T449 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/'
   [junit4]   2> 168521 T449 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 168524 T449 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 168538 T449 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 168567 T449 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 168624 T449 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 168626 T449 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 168640 T449 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 169025 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 169039 T449 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 169041 T449 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 169054 T449 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 169058 T449 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 169061 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 169062 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 169062 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 169062 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 169063 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 169063 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 169063 T449 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 169064 T449 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 169064 T449 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 169064 T449 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f3fd958
   [junit4]   2> 169065 T449 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data
   [junit4]   2> 169065 T449 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 169065 T449 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 169065 T449 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 169066 T449 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=73840210, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 169067 T449 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1abcd676 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@63a3eab0),segFN=segments_1,generation=1}
   [junit4]   2> 169067 T449 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 169073 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 169073 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 169073 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 169074 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 169074 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 169074 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 169074 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 169075 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 169075 T449 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 169094 T449 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169096 T449 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169097 T449 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169099 T449 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169105 T449 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/replication,/get,/admin/mbeans,/admin/properties,/update/csv,/update/json,/admin/logging,/admin/system,/admin/threads,/update,/admin/ping,/schema,/admin/plugins,/update/json/docs,/admin/segments,/config,standard,/admin/file
   [junit4]   2> 169107 T449 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 169108 T449 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 169108 T449 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 169110 T449 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 169110 T449 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 169111 T449 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=38, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10602606676023907]
   [junit4]   2> 169111 T449 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1abcd676 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@63a3eab0),segFN=segments_1,generation=1}
   [junit4]   2> 169111 T449 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 169111 T449 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6a3c49d3[collection1] main
   [junit4]   2> 169112 T449 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 169113 T449 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 169113 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 169113 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 169113 T449 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 169114 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 169114 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 169114 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 169114 T449 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 169115 T449 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 169116 T450 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6a3c49d3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 169116 T449 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 169117 T453 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32131 collection:collection1 shard:shard1
   [junit4]   2> 169117 T395 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 169118 T395 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 169118 T453 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 169122 T453 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 169123 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 169124 T453 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 169124 T453 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C108 name=collection1 org.apache.solr.core.SolrCore@210cd041 url=http://127.0.0.1:32131/collection1 node=127.0.0.1:32131_ C108_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:32131, node_name=127.0.0.1:32131_, state=down}
   [junit4]   2> 169124 T453 C:collection1 S:shard1 C108 P32131 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32131/collection1/
   [junit4]   2> 169124 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 169125 T453 C:collection1 S:shard1 C108 P32131 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 169125 T453 C:collection1 S:shard1 C108 P32131 oasc.SyncStrategy.syncToMe http://127.0.0.1:32131/collection1/ has no replicas
   [junit4]   2> 169125 T453 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32131/collection1/ shard1
   [junit4]   2> 169125 T453 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 169126 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169126 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169126 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169130 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 169131 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 169234 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169234 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169234 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169283 T453 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:32131/collection1/ and leader is http://127.0.0.1:32131/collection1/
   [junit4]   2> 169284 T453 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32131
   [junit4]   2> 169284 T453 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 169284 T453 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 169284 T453 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 169285 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 169286 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32131_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"} current state version: 7
   [junit4]   2> 169287 T422 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32131_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 169390 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169390 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169390 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstat

[...truncated too long message...]

ue
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1504270 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:31600/_bkj/mq
   [junit4]   2> 1504271 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:31574/_bkj/mq
   [junit4]   2> 1504271 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:31559/_bkj/mq
   [junit4]   2> 1504271 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31585/_bkj/mq
   [junit4]   2> 1504271 T3781 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1504272 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31600/_bkj/mq
   [junit4]   2> 1504272 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31574/_bkj/mq
   [junit4]   2> 1504272 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31559/_bkj/mq
   [junit4]   2> 1504272 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31585/_bkj/mq
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=998831C6A7E2310A -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ro -Dtests.timezone=SystemV/EST5 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   80.1s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([998831C6A7E2310A:11DC0E1C091E5CF2]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:31600 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1504288 T3781 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 998831C6A7E2310A-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ro, timezone=SystemV/EST5
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=46987136,total=306708480
   [junit4]   2> NOTE: All tests run in this JVM: [DeleteReplicaTest, TestDynamicFieldResource, SearchHandlerTest, SpellPossibilityIteratorTest, TestSolrCoreProperties, ShardRoutingTest, ExternalFileFieldSortTest, TestAnalyzeInfixSuggestions, RequiredFieldsTest, TestRTGBase, LeaderFailoverAfterPartitionTest, XmlUpdateRequestHandlerTest, LeaderElectionIntegrationTest, DeleteInactiveReplicaTest, TestEmbeddedSolrServerConstructors, AddBlockUpdateTest, TestHashQParserPlugin, TestManagedSchema, TestManagedStopFilterFactory, DocumentAnalysisRequestHandlerTest, TestCSVLoader, TestCursorMarkWithoutUniqueKey, FastVectorHighlighterTest, TestDistributedGrouping, BadIndexSchemaTest, HardAutoCommitTest, TestRawResponseWriter, DateMathParserTest, InfoHandlerTest, UnloadDistributedZkTest, TestMacros, TestLMDirichletSimilarityFactory, DocValuesMissingTest, TestSchemaSimilarityResource, TestConfigSets, TestTrackingShardHandlerFactory, SolrXmlInZkTest, TestAnalyzedSuggestions, ChaosMonkeySafeLeaderTest, TestHashPartitioner, TestSchemaVersionResource, TestReqParamsAPI, SliceStateTest, SegmentsInfoRequestHandlerTest, RAMDirectoryFactoryTest, LukeRequestHandlerTest, DistributedFacetPivotLongTailTest, HdfsCollectionsAPIDistributedZkTest, TestPartialUpdateDeduplication, DistanceUnitsTest, MBeansHandlerTest, TestFiltering, TestInfoStreamLogging, TestCollationFieldDocValues, TestLuceneMatchVersion, TestRandomFaceting, DistributedQueueTest, RecoveryZkTest, OpenCloseCoreStressTest, BasicDistributedZkTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 80.20s, 1 test, 1 error <<< FAILURES!

[...truncated 712 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 216312738 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2903 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2903/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:34770/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:34770/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([9D5B81451A89ABB3:150FBE9FB475C64B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:34770 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond
	at __randomizedtesting.SeedInfo.seed([9D5B81451A89ABB3:150FBE9FB475C64B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:54113 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:54113 failed to respond
	at __randomizedtesting.SeedInfo.seed([9D5B81451A89ABB3:150FBE9FB475C64B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:54113 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9344 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/init-core-data-001
   [junit4]   2> 101702 T157 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 101778 T157 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 101792 T158 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 102196 T157 oasc.ZkTestServer.run start zk server on port:54100
   [junit4]   2> 102393 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 102703 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 102978 T165 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4dcbbdda name:ZooKeeperConnection Watcher:127.0.0.1:54100 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 102978 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 102979 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 102998 T157 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 103192 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 103193 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 103198 T168 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e891f79 name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 103199 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 103200 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 103227 T157 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 103232 T157 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 103236 T157 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 103239 T157 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 103243 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 103244 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 103250 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 103251 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 103256 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 103256 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 103261 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 103261 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 103265 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 103265 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 103269 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 103270 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 103300 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 103301 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 103305 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 103306 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 103309 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 103310 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 103314 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 103314 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 103318 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 103318 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 105888 T157 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1
   [junit4]   2> 105895 T157 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 105899 T157 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54106
   [junit4]   2> 105899 T157 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/tempDir-001/control/data, hostContext=/, hostPort=54105}
   [junit4]   2> 105908 T157 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 105908 T157 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/'
   [junit4]   2> 105978 T157 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/solr.xml
   [junit4]   2> 106074 T157 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores
   [junit4]   2> 106074 T157 oasc.CoreContainer.<init> New CoreContainer 470219515
   [junit4]   2> 106074 T157 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/]
   [junit4]   2> 106075 T157 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/lib
   [junit4]   2> 106076 T157 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/lib).
   [junit4]   2> 106089 T157 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 106089 T157 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 106089 T157 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 106090 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 106090 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 106090 T157 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 106090 T157 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 106091 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 106091 T157 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 106091 T157 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 106092 T157 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 106099 T157 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 106100 T157 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 106100 T157 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 106101 T157 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 106101 T157 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54100/solr
   [junit4]   2> 106158 T157 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 106159 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 106160 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 106162 T180 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@220fe47c name:ZooKeeperConnection Watcher:127.0.0.1:54100 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 106163 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 106163 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 106184 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 106186 T183 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f2d3e9d name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 106186 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 106211 T157 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 106229 T157 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 106239 T157 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 106244 T157 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 106248 T157 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 106287 T157 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 106290 T157 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 106293 T157 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 106295 T157 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54105_
   [junit4]   2> 106314 T157 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54105_
   [junit4]   2> 106318 T157 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 106322 T157 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 106325 T157 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 106364 T157 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:54105_
   [junit4]   2> 106365 T157 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 106368 T157 oasc.Overseer.start Overseer (id=93610334489739267-127.0.0.1:54105_-n_0000000000) starting
   [junit4]   2> 106410 T157 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 106473 T157 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 106512 T185 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 106512 T157 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 106546 T184 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 106576 T157 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores
   [junit4]   2> 106607 T157 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, coreNodeName=, shard=, name=collection1, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1, loadOnStartup=true, dataDir=data/, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/, collection=control_collection}
   [junit4]   2> 106607 T157 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/
   [junit4]   2> 106608 T157 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 106609 T187 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 106610 T187 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 106633 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 106633 T187 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 106642 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:54105_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 106722 T184 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:54105_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 106733 T184 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 106816 T184 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 106843 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 107634 T187 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 107634 T187 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 107635 T187 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 107635 T187 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 107637 T187 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 107637 T187 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/'
   [junit4]   2> 107677 T187 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 107698 T187 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 107728 T187 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 107752 T187 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 107797 T187 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 107799 T187 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 107817 T187 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 108335 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 108376 T187 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 108378 T187 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 108399 T187 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 108424 T187 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 108471 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 108473 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 108473 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 108474 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 108475 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 108475 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 108476 T187 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 108476 T187 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 108476 T187 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data/
   [junit4]   2> 108477 T187 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@44a0c74c
   [junit4]   2> 108488 T187 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data
   [junit4]   2> 108488 T187 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data/index/
   [junit4]   2> 108488 T187 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 108489 T187 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data/index
   [junit4]   2> 108489 T187 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=14, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 108490 T187 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1892ebd6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55b93b34),segFN=segments_1,generation=1}
   [junit4]   2> 108490 T187 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 108498 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 108605 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 108606 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 108606 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 108606 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 108606 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 108607 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 108607 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 108607 T187 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 108633 T187 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108643 T187 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108645 T187 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108648 T187 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108659 T187 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/segments,standard,/admin/logging,/admin/plugins,/admin/system,/admin/file,/get,/update/json/docs,/admin/luke,/admin/mbeans,/admin/ping,/schema,/replication,/config,/update/json,/admin/properties,/update/csv,/admin/threads
   [junit4]   2> 108663 T187 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 108665 T187 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 108666 T187 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 108667 T187 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 108668 T187 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 108669 T187 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.792881320471249]
   [junit4]   2> 108669 T187 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1892ebd6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55b93b34),segFN=segments_1,generation=1}
   [junit4]   2> 108669 T187 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 108670 T187 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@658eb67b[collection1] main
   [junit4]   2> 108670 T187 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 108672 T187 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 108672 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 108690 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 108690 T187 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 108690 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 108691 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 108691 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 108692 T187 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 108692 T187 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 108695 T188 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@658eb67b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 108705 T187 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 108706 T191 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54105 collection:control_collection shard:shard1
   [junit4]   2> 108707 T157 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 108708 T157 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 108758 T191 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 108768 T191 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 108771 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108772 T191 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 108772 T191 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 108773 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C30 name=collection1 org.apache.solr.core.SolrCore@211f7810 url=http://127.0.0.1:54105/collection1 node=127.0.0.1:54105_ C30_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:54105_, base_url=http://127.0.0.1:54105}
   [junit4]   2> 108786 T191 C:control_collection S:shard1 C30 P54105 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54105/collection1/
   [junit4]   2> 108787 T191 C:control_collection S:shard1 C30 P54105 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 108787 T191 C:control_collection S:shard1 C30 P54105 oasc.SyncStrategy.syncToMe http://127.0.0.1:54105/collection1/ has no replicas
   [junit4]   2> 108788 T191 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54105/collection1/ shard1
   [junit4]   2> 108789 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108807 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 108808 T191 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 108808 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 108818 T195 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40b5a9b2 name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 108818 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 108819 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 108819 T157 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 108823 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108825 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 108827 T157 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 108827 T157 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 108828 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 108828 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 108830 T198 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f3e04c9 name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 108831 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 108831 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 108834 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108836 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 108836 T184 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 108837 T195 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108837 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108875 T191 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:54105/collection1/ and leader is http://127.0.0.1:54105/collection1/
   [junit4]   2> 108876 T191 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54105
   [junit4]   2> 108876 T191 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 108876 T191 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 108876 T191 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 108878 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108880 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:54105_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 108881 T184 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:54105_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 108882 T184 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 108884 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108884 T195 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108992 T195 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108992 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 110958 T157 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1
   [junit4]   2> 110962 T157 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001
   [junit4]   2> 110963 T157 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 110966 T157 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54114
   [junit4]   2> 110967 T157 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=54113, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/tempDir-001/jetty1}
   [junit4]   2> 110968 T157 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 110968 T157 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/'
   [junit4]   2> 111057 T157 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/solr.xml
   [junit4]   2> 111148 T157 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores
   [junit4]   2> 111148 T157 oasc.CoreContainer.<init> New CoreContainer 1308631172
   [junit4]   2> 111149 T157 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/]
   [junit4]   2> 111149 T157 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/lib
   [junit4]   2> 111160 T157 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/lib).
   [junit4]   2> 111175 T157 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 111176 T157 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 111176 T157 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 111176 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 111177 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 111177 T157 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 111178 T157 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 111178 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 111178 T157 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 111179 T157 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 111179 T157 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 111191 T157 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 111192 T157 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 111193 T157 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 111193 T157 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 111193 T157 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54100/solr
   [junit4]   2> 111194 T157 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 111194 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 111195 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 111198 T210 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45150b1b name:ZooKeeperConnection Watcher:127.0.0.1:54100 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 111198 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 111199 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 111203 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 111205 T213 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@30aa7379 name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 111205 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 111215 T157 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 112227 T157 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54113_
   [junit4]   2> 112229 T157 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54113_
   [junit4]   2> 112234 T157 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 112243 T157 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores
   [junit4]   2> 112245 T157 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, schema=schema.xml, shard=, loadOnStartup=true, dataDir=data/, name=collection1, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1, transient=false}
   [junit4]   2> 112246 T157 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/
   [junit4]   2> 112246 T157 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 112250 T214 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 112250 T214 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 112252 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 112254 T214 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 112254 T214 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 112254 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:54113_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 112255 T214 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 112255 T184 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:54113_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 112256 T184 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 112256 T184 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 112259 T184 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 112264 T214 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 112265 T214 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 112265 T214 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 112265 T214 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 112266 T214 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 112266 T214 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 112267 T214 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 112267 T214 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/'
   [junit4]   2> 112285 T214 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 112289 T214 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 112300 T214 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 112338 T214 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 112388 T214 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 112390 T214 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 112401 T214 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 112910 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 112927 T214 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 112929 T214 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 112948 T214 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 112957 T214 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 112979 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 112980 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 112980 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 112981 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 112982 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 112982 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 112982 T214 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 112982 T214 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 112983 T214 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 112983 T214 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@44a0c74c
   [junit4]   2> 112984 T214 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data
   [junit4]   2> 112984 T214 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 112984 T214 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 112985 T214 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 112985 T214 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=14, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 112986 T214 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@adbe555 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6aad3504),segFN=segments_1,generation=1}
   [junit4]   2> 112986 T214 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 112996 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 112997 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 112997 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 112997 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 112997 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 112998 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 112998 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 112999 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 112999 T214 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 113013 T214 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113016 T214 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113019 T214 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113021 T214 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113049 T214 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/segments,standard,/admin/logging,/admin/plugins,/admin/system,/admin/file,/get,/update/json/docs,/admin/luke,/admin/mbeans,/admin/ping,/schema,/replication,/config,/update/json,/admin/properties,/update/csv,/admin/threads
   [junit4]   2> 113051 T214 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 113052 T214 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 113053 T214 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 113054 T214 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 113054 T214 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 113056 T214 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.792881320471249]
   [junit4]   2> 113056 T214 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@adbe555 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6aad3504),segFN=segments_1,generation=1}
   [junit4]   2> 113057 T214 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 113057 T214 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@578a1898[collection1] main
   [junit4]   2> 113057 T214 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 113059 T214 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 113059 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 113060 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 113060 T214 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 113060 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 113061 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 113061 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 113061 T214 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 113062 T214 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 113072 T215 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@578a1898[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 113073 T214 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 113074 T218 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54113 collection:collection1 shard:shard3
   [junit4]   2> 113075 T157 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 113075 T157 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 113075 T218 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 113083 T218 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 113086 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 113087 T218 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 113087 T218 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C31 name=collection1 org.apache.solr.core.SolrCore@729f3241 url=http://127.0.0.1:54113/collection1 node=127.0.0.1:54113_ C31_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:54113_, base_url=http://127.0.0.1:54113}
   [junit4]   2> 113088 T218 C:collection1 S:shard3 C31 P54113 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54113/collection1/
   [junit4]   2> 113088 T218 C:collection1 S:shard3 C31 P54113 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 113088 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 113088 T218 C:collection1 S:shard3 C31 P54113 oasc.SyncStrategy.syncToMe http://127.0.0.1:54113/collection1/ has no replicas
   [junit4]   2> 113089 T218 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54113/collection1/ shard3
   [junit4]   2> 113089 T218 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 113092 T184 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 113093 T213 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 113095 T213 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 113101 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 113104 T184 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 113105 T213 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 113106 T213 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 113111 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 113151 T218 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:54113/collection1/ and leader is http://127.0.0.1:54113/collection1/
   [junit4]   2> 113152 T218 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54113
   [junit4]   2> 113153 T218 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 113153 T218 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 113153 T218 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 113156 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 113159 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:54113_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 113160 T184 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:54113_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null}
   [junit4] 

[...truncated too long message...]

0406174556825
   [junit4]   2> 1762047 T3781 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1762049 T3670 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/fs,null}
   [junit4]   2> 1762225 T3670 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:34962 34962
   [junit4]   2> 1762569 T3671 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:34962 34962
   [junit4]   2> 1762570 T3671 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1762571 T3670 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:34979/fs
   [junit4]   2> 1762571 T3670 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:34969/fs
   [junit4]   2> 1762571 T3670 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:34986/fs
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=9D5B81451A89ABB3 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es_ES -Dtests.timezone=Pacific/Midway -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   71.9s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9D5B81451A89ABB3:150FBE9FB475C64B]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1762587 T3670 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 9D5B81451A89ABB3-001
   [junit4]   2> 71897 T3669 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=es_ES, timezone=Pacific/Midway
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=185934920,total=312475648
   [junit4]   2> NOTE: All tests run in this JVM: [TestUniqueKeyFieldResource, OutOfBoxZkACLAndCredentialsProvidersTest, SystemInfoHandlerTest, TestReplicaProperties, ZkStateWriterTest, TestHdfsUpdateLog, TestBinaryResponseWriter, TestWriterPerf, FastVectorHighlighterTest, TestLRUCache, TestStressVersions, TestAddFieldRealTimeGet, PrimUtilsTest, MinimalSchemaTest, OverseerCollectionProcessorTest, SuggesterFSTTest, BlockCacheTest, MBeansHandlerTest, PrimitiveFieldTypeTest, TestManagedResource, TestRecovery, HttpPartitionTest, RequestHandlersTest, BadComponentTest, TestJettySolrRunner, CachingDirectoryFactoryTest, TestCloudPivotFacet, CustomCollectionTest, TestNRTOpen, OutputWriterTest, TestConfigSets, TestArbitraryIndexDir, TestFastOutputStream, RecoveryZkTest, TestHighFrequencyDictionaryFactory, BinaryUpdateRequestHandlerTest, TestManagedSchemaFieldTypeResource, TestExactSharedStatsCache, BasicZkTest, TestSort, SolrTestCaseJ4Test, TestDocBasedVersionConstraints, TestSolrQueryParserResource, DefaultValueUpdateProcessorTest, DirectSolrSpellCheckerTest, DeleteLastCustomShardedReplicaTest, ClusterStateUpdateTest, TestRangeQuery, TestExpandComponent, MergeStrategyTest, TestLMDirichletSimilarityFactory, TestSolrDynamicMBean, TestConfig, SimplePostToolTest, TestBulkSchemaConcurrent, TestIntervalFaceting, DistributedQueryComponentOptimizationTest, TestMacros, HdfsUnloadDistributedZkTest, FullSolrCloudDistribCmdsTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 72.68s, 1 test, 1 error <<< FAILURES!

[...truncated 723 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 60 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 227544524 bytes
Compression is 0.0%
Took 45 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2902 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2902/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:62732/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62732/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([26BC33507B4A9FC1:AEE80C8AD5B6F239]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:62732 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond
	at __randomizedtesting.SeedInfo.seed([26BC33507B4A9FC1:AEE80C8AD5B6F239]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:62804 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:62804 failed to respond
	at __randomizedtesting.SeedInfo.seed([26BC33507B4A9FC1:AEE80C8AD5B6F239]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:62804 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10002 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/init-core-data-001
   [junit4]   2> 1486847 T4980 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1486852 T4980 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1486852 T4981 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1486952 T4980 oasc.ZkTestServer.run start zk server on port:62708
   [junit4]   2> 1486953 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1486953 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1486957 T4988 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@37840492 name:ZooKeeperConnection Watcher:127.0.0.1:62708 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1486957 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1486957 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1486957 T4980 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1486959 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1486960 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1486961 T4991 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e4dd35c name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1486962 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1486962 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1486962 T4980 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1486964 T4980 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1486965 T4980 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1486966 T4980 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1486967 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1486968 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1486969 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1486970 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1487073 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1487073 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1487075 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1487075 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1487077 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1487077 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1487079 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1487079 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1487081 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1487081 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1487083 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1487083 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1487085 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1487085 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1487087 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1487087 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1487089 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1487089 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1488674 T4980 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1
   [junit4]   2> 1488678 T4980 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1488682 T4980 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62717
   [junit4]   2> 1488683 T4980 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=62716, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/tempDir-001/control/data}
   [junit4]   2> 1488683 T4980 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 1488683 T4980 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/'
   [junit4]   2> 1488727 T4980 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/solr.xml
   [junit4]   2> 1488781 T4980 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores
   [junit4]   2> 1488782 T4980 oasc.CoreContainer.<init> New CoreContainer 1635486708
   [junit4]   2> 1488782 T4980 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/]
   [junit4]   2> 1488782 T4980 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/lib
   [junit4]   2> 1488783 T4980 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/lib).
   [junit4]   2> 1488792 T4980 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1488793 T4980 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1488793 T4980 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1488793 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1488793 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1488795 T4980 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1488802 T4980 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1488803 T4980 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1488803 T4980 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1488803 T4980 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1488804 T4980 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:62708/solr
   [junit4]   2> 1488804 T4980 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1488804 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1488805 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1488806 T5003 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7be47528 name:ZooKeeperConnection Watcher:127.0.0.1:62708 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1488807 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1488807 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1488809 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1488810 T5006 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c304ccf name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1488810 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1488811 T4980 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1488813 T4980 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1488814 T4980 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1488815 T4980 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1488817 T4980 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1488818 T4980 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1488819 T4980 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1488820 T4980 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1488821 T4980 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62716_
   [junit4]   2> 1488821 T4980 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62716_
   [junit4]   2> 1488822 T4980 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1488823 T4980 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1488824 T4980 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1488825 T4980 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:62716_
   [junit4]   2> 1488825 T4980 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1488826 T4980 oasc.Overseer.start Overseer (id=93609472381485059-127.0.0.1:62716_-n_0000000000) starting
   [junit4]   2> 1488827 T4980 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1488831 T4980 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1488832 T5008 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1488832 T4980 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1488832 T5007 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1488835 T4980 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores
   [junit4]   2> 1488837 T4980 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1, collection=control_collection, schema=schema.xml, coreNodeName=, shard=, loadOnStartup=true, config=solrconfig.xml, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/}
   [junit4]   2> 1488837 T4980 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/
   [junit4]   2> 1488837 T4980 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1488838 T5010 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1488838 T5010 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1488839 T5010 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1488839 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1488839 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "node_name":"127.0.0.1:62716_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 1488840 T5007 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "node_name":"127.0.0.1:62716_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 1488840 T5007 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1488840 T5007 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1488841 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1489839 T5010 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1489840 T5010 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1489841 T5010 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1489841 T5010 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1489841 T5010 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1489842 T5010 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/'
   [junit4]   2> 1489860 T5010 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1489862 T5010 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1489873 T5010 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1489897 T5010 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1489950 T5010 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1490052 T5010 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1490063 T5010 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1490500 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1490516 T5010 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1490519 T5010 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1490528 T5010 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1490533 T5010 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1490537 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1490537 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1490538 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1490538 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1490539 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1490539 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1490539 T5010 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1490539 T5010 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1490540 T5010 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data/
   [junit4]   2> 1490540 T5010 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78e9fb08
   [junit4]   2> 1490541 T5010 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data
   [junit4]   2> 1490541 T5010 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1490541 T5010 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1490542 T5010 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data/index
   [junit4]   2> 1490542 T5010 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1490543 T5010 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@cbce229 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@44a3f847),segFN=segments_1,generation=1}
   [junit4]   2> 1490543 T5010 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1490549 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1490549 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1490550 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1490550 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1490550 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1490550 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1490551 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1490551 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1490552 T5010 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1490574 T5010 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1490577 T5010 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1490579 T5010 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1490581 T5010 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1490587 T5010 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/replication,/admin/system,/admin/properties,/update/json,/admin/file,/update,/admin/logging,/config,/update/json/docs,/admin/plugins,/admin/ping,/admin/threads,/admin/mbeans,/admin/luke,/get,/admin/segments,/schema,/update/csv
   [junit4]   2> 1490589 T5010 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1490590 T5010 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1490591 T5010 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1490592 T5010 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1490592 T5010 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1490593 T5010 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=92.4228515625, floorSegmentMB=0.3427734375, forceMergeDeletesPctAllowed=1.1550384131963098, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.49791742044625653
   [junit4]   2> 1490594 T5010 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@cbce229 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@44a3f847),segFN=segments_1,generation=1}
   [junit4]   2> 1490594 T5010 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1490594 T5010 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@460fdd6d[collection1] main
   [junit4]   2> 1490595 T5010 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1490595 T5010 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1490596 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1490596 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1490596 T5010 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1490596 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1490597 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1490597 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1490597 T5010 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1490598 T5010 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1490598 T5011 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@460fdd6d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1490599 T5010 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1490600 T5014 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:62716 collection:control_collection shard:shard1
   [junit4]   2> 1490600 T4980 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1490600 T4980 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1490600 T5014 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1490602 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1490603 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1490604 T5014 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1490604 T5018 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44539bd3 name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1490605 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1490605 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1490605 T4980 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1490605 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1490606 T5014 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1490606 T5014 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C660 name=collection1 org.apache.solr.core.SolrCore@4ecb64b6 url=http://127.0.0.1:62716/collection1 node=127.0.0.1:62716_ C660_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:62716, node_name=127.0.0.1:62716_, state=down}
   [junit4]   2> 1490607 T5014 C:control_collection S:shard1 C660 P62716 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:62716/collection1/
   [junit4]   2> 1490607 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1490607 T5014 C:control_collection S:shard1 C660 P62716 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1490607 T5014 C:control_collection S:shard1 C660 P62716 oasc.SyncStrategy.syncToMe http://127.0.0.1:62716/collection1/ has no replicas
   [junit4]   2> 1490607 T5014 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:62716/collection1/ shard1
   [junit4]   2> 1490608 T5014 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1490608 T4980 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1490608 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490608 T4980 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1490608 T5018 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490609 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1490610 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1490611 T5021 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b9870e0 name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1490612 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1490612 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1490612 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1490613 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1490615 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1490615 T5007 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1490616 T5018 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490616 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490618 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1490618 T5007 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1490619 T5018 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490619 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490663 T5014 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:62716/collection1/ and leader is http://127.0.0.1:62716/collection1/
   [junit4]   2> 1490664 T5014 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:62716
   [junit4]   2> 1490664 T5014 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1490664 T5014 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1490664 T5014 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1490666 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1490667 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "node_name":"127.0.0.1:62716_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1"} current state version: 4
   [junit4]   2> 1490667 T5007 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "node_name":"127.0.0.1:62716_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 1490771 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490771 T5018 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1492134 T4980 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1
   [junit4]   2> 1492136 T4980 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001
   [junit4]   2> 1492137 T4980 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1492139 T4980 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62733
   [junit4]   2> 1492139 T4980 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=62732, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores}
   [junit4]   2> 1492140 T4980 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 1492140 T4980 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/'
   [junit4]   2> 1492176 T4980 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/solr.xml
   [junit4]   2> 1492219 T4980 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores
   [junit4]   2> 1492220 T4980 oasc.CoreContainer.<init> New CoreContainer 125063838
   [junit4]   2> 1492220 T4980 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/]
   [junit4]   2> 1492220 T4980 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/lib
   [junit4]   2> 1492221 T4980 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/lib).
   [junit4]   2> 1492231 T4980 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1492231 T4980 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1492231 T4980 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1492231 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1492233 T4980 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1492233 T4980 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1492238 T4980 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1492239 T4980 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1492239 T4980 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1492240 T4980 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1492240 T4980 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:62708/solr
   [junit4]   2> 1492240 T4980 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1492240 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1492241 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1492243 T5033 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@872ac36 name:ZooKeeperConnection Watcher:127.0.0.1:62708 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1492244 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1492244 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1492247 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1492248 T5036 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@884ee6c name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1492248 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1492252 T4980 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1493256 T4980 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62732_
   [junit4]   2> 1493257 T4980 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62732_
   [junit4]   2> 1493260 T4980 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1493261 T4980 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores
   [junit4]   2> 1493264 T4980 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, coreNodeName=, transient=false, schema=schema.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/, shard=, dataDir=data/, loadOnStartup=true, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1}
   [junit4]   2> 1493264 T4980 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/
   [junit4]   2> 1493264 T4980 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1493266 T5037 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1493266 T5037 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1493267 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1493267 T5037 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1493267 T5037 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1493267 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62732",
   [junit4]   2> 	  "node_name":"127.0.0.1:62732_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null} current state version: 5
   [junit4]   2> 1493268 T5037 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1493268 T5007 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62732",
   [junit4]   2> 	  "node_name":"127.0.0.1:62732_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 1493268 T5007 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 1493268 T5007 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1493268 T5037 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1493268 T5037 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1493269 T5007 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1493269 T5036 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1493270 T5036 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1494269 T5037 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1494270 T5037 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1494270 T5037 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1494270 T5037 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1494271 T5037 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1494271 T5037 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1494284 T5037 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1494286 T5037 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1494294 T5037 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1494324 T5037 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1494362 T5037 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1494464 T5037 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1494472 T5037 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1494762 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1494774 T5037 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1494776 T5037 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1494784 T5037 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1494787 T5037 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1494790 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1494791 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1494791 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1494791 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1494792 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1494792 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1494792 T5037 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1494793 T5037 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1494793 T5037 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1494793 T5037 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78e9fb08
   [junit4]   2> 1494794 T5037 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1494794 T5037 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1494794 T5037 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1494795 T5037 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1494795 T5037 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1494796 T5037 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4d8423ef lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1e5347ab),segFN=segments_1,generation=1}
   [junit4]   2> 1494796 T5037 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1494801 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1494801 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1494803 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1494803 T5037 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1494822 T5037 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494824 T5037 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494826 T5037 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494827 T5037 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494834 T5037 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/replication,/admin/system,/admin/properties,/update/json,/admin/file,/update,/admin/logging,/config,/update/json/docs,/admin/plugins,/admin/ping,/admin/threads,/admin/mbeans,/admin/luke,/get,/admin/segments,/schema,/update/csv
   [junit4]   2> 1494836 T5037 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1494838 T5037 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1494838 T5037 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1494839 T5037 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1494839 T5037 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1494840 T5037 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=92.4228515625, floorSegmentMB=0.3427734375, forceMergeDeletesPctAllowed=1.1550384131963098, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.49791742044625653
   [junit4]   2> 1494841 T5037 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4d8423ef lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1e5347ab),segFN=segments_1,generation=1}
   [junit4]   2> 1494841 T5037 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1494841 T5037 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@56a3f1e1[collection1] main
   [junit4]   2> 1494842 T5037 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1494842 T5037 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1494843 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1494843 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1494843 T5037 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1494843 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1494844 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1494844 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1494844 T5037 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1494844 T5037 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1494845 T5038 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@56a3f1e1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1494845 T5037 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1494846 T5041 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:62732 collection:collection1 shard:shard1
   [junit4]   2> 1494846 T4980 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1494847 T4980 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1494847 T5041 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1494850 T5041 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1494852 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1494852 T5041 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1494852 T5041 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C661 name=collection1 org.apache.solr.core.SolrCore@55daa5ce url=http://127.0.0.1:62732/collection1 node=127.0.0.1:62732_ C661_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:62732, node_name=127.0.0.1:62732_, state=down}
   [junit4]   2> 1494852 T5041 C:collection1 S:shard1 C661 P62732 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:62732/collection1/
   [junit4]   2> 1494852 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1494853 T5041 C:collection1 S:shard1 C661 P62732 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1494853 T5041 C:collection1 S:shard1 C661 P62732 oasc.SyncStrategy.syncToMe http://127.0.0.1:62732/collection1/ has no replicas
   [junit4]   2> 1494853 T5041 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:62732/collection1/ shard1
   [junit4]   2> 1494853 T5041 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1494853 T5007 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1494854 T5036 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1494855 T5036 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1494856 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1494857 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62732",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1494858 T5007 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1494858 T5036 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1494859 T5036 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1494907 T5041 C:collection1 S:shard1

[...truncated too long message...]

lk to ZK
   [junit4]   2> 1633632 T3227 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1633736 T3227 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:62998 62998
   [junit4]   2> 1634082 T3228 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:62998 62998
   [junit4]   2> 1634085 T3228 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1634085 T3227 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63011/
   [junit4]   2> 1634086 T3227 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:63007/
   [junit4]   2> 1634086 T3227 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63001/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=26BC33507B4A9FC1 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=fr_BE -Dtests.timezone=America/Fortaleza -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   71.9s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([26BC33507B4A9FC1:AEE80C8AD5B6F239]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1634102 T3227 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 26BC33507B4A9FC1-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), multiDefault=Lucene50(blocksize=128), id=PostingsFormat(name=LuceneFixedGap), _version_=FSTOrd50, range_facet_l=Lucene50(blocksize=128), intDefault=FSTOrd50, text=FST50}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=DefaultSimilarity, locale=fr_BE, timezone=America/Fortaleza
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=97899896,total=327680000
   [junit4]   2> NOTE: All tests run in this JVM: [TestLRUStatsCache, DateRangeFieldTest, LukeRequestHandlerTest, CacheHeaderTest, TestCloudInspectUtil, TestCharFilters, WordBreakSolrSpellCheckerTest, TestJsonRequest, TestPostingsSolrHighlighter, TestBinaryField, TestQueryUtils, TestFieldTypeCollectionResource, TestConfigOverlay, TestFuzzyAnalyzedSuggestions, MultiThreadedOCPTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, DistanceFunctionTest, TestConfig, BlockDirectoryTest, SimpleCollectionCreateDeleteTest, PreAnalyzedUpdateProcessorTest, CursorMarkTest, TestMiniSolrCloudCluster, DeleteLastCustomShardedReplicaTest, AnalyticsQueryTest, SolrInfoMBeanTest, TestCloudManagedSchema, TestDynamicFieldResource, TestIntervalFaceting, AssignTest, HdfsDirectoryFactoryTest, CustomCollectionTest, TestLMDirichletSimilarityFactory, TestLFUCache, DocExpirationUpdateProcessorFactoryTest, HttpPartitionTest, SharedFSAutoReplicaFailoverUtilsTest, ChangedSchemaMergeTest, TestAnalyzedSuggestions, VMParamsZkACLAndCredentialsProvidersTest, TestPHPSerializedResponseWriter, TestFieldCollectionResource, TestInitQParser, HdfsBasicDistributedZkTest, TestZkChroot, CursorPagingTest, RemoteQueryErrorTest, ReturnFieldsTest, ScriptEngineTest, TestCodecSupport, DistributedSuggestComponentTest, TestShardHandlerFactory, BadComponentTest, UpdateRequestProcessorFactoryTest, TestReqParamsAPI, TestImplicitCoreProperties, TestHashPartitioner, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 71.97s, 1 test, 1 error <<< FAILURES!

[...truncated 722 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 45 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 228365171 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2901 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2901/

4 tests failed.
REGRESSION:  org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds

Error Message:
soft529 wasn't fast enough

Stack Trace:
java.lang.AssertionError: soft529 wasn't fast enough
	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:5FA022F5B43939E3]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:25435/_f/em/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25435/_f/em/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:8620E4AFABB664BC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:25435 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:41448 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:41448 failed to respond
	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:8620E4AFABB664BC]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:41448 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond
	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:8620E4AFABB664BC]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9228 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001
   [junit4]   2> 15934 T16 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 15936 T16 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 15936 T16 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 15938 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 15939 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 15940 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 16107 T16 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 16199 T16 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 16345 T16 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 16347 T16 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 16454 T16 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 19050 T16 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 19180 T16 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 19190 T16 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 20464 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 20491 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 20497 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 20503 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 20504 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 20504 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 20505 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 20506 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 20519 T16 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 20520 T16 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 20520 T16 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 20562 T16 oasc.CoreContainer.<init> New CoreContainer 1713272514
   [junit4]   2> 20563 T16 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 20563 T16 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib
   [junit4]   2> 20564 T16 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib).
   [junit4]   2> 20581 T16 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 20582 T16 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 20582 T16 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 20583 T16 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 20583 T16 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 20584 T16 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 20584 T16 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 20585 T16 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 20585 T16 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 20586 T16 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 20586 T16 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 20595 T16 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 20597 T16 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 20598 T16 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 20598 T16 oasc.CoreContainer.load Node Name: testNode
   [junit4]   2> 20601 T16 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, loadOnStartup=true, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, transient=false, collection=collection1, schema=schema.xml, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001, instanceDir=collection1, shard=shard1}
   [junit4]   2> 20602 T17 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 20604 T17 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 20605 T17 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 20605 T17 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 20720 T17 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 20756 T17 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 20856 T17 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 20858 T17 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 20956 T17 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 21430 T17 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 21456 T17 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 21459 T17 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 21485 T17 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 21491 T17 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 21495 T17 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 21496 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 21497 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 21497 T17 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 21498 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 21498 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 21499 T17 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 21499 T17 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 21500 T17 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/
   [junit4]   2> 21508 T17 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 21508 T17 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001
   [junit4]   2> 21509 T17 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index/
   [junit4]   2> 21509 T17 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 21510 T17 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index
   [junit4]   2> 21511 T17 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 21519 T17 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_1,generation=1}
   [junit4]   2> 21520 T17 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 21856 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 21973 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 21981 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 21982 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 21982 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 21983 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 21983 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 22017 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 22018 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 22335 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 22336 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 22337 T17 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 22374 T17 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 22378 T17 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 22381 T17 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 22384 T17 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 22689 T17 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 22815 T17 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json,/replication,spellCheckCompRH_Direct,/get,mock,/search-facet-invariants,mltrh,spellCheckWithWordbreak_Direct,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,dismax,/search-facet-def,spellCheckWithWordbreak,defaults,/admin/segments,/admin/threads,/config,tvrh,/admin/properties,standard,/debug/dump,/update/json/docs,lazy,/mlt,spellCheckCompRH1,/schema,/terms,spellCheckCompRH,/admin/system,/admin/logging,/update,/admin/luke
   [junit4]   2> 22827 T17 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 22831 T17 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 22965 T17 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 22966 T17 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 23032 T17 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 23033 T17 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 23034 T17 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=98.2822265625, floorSegmentMB=1.3701171875, forceMergeDeletesPctAllowed=26.756904425149433, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8147500519206919
   [junit4]   2> 23035 T17 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_1,generation=1}
   [junit4]   2> 23036 T17 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 23037 T17 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@785e5b21[collection1] main
   [junit4]   2> 23056 T17 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 23057 T17 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 23058 T17 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 23059 T17 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 23542 T17 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 24040 T17 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 24041 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 24045 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 24045 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 24046 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 24049 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 24052 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 24054 T17 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 24054 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 24056 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 24058 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 24059 T18 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@785e5b21[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 24092 T16 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 24107 T16 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 24108 T16 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 24111 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 24112 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 24113 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 24359 T16 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 24424 T16 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 24554 T16 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 24555 T16 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 24619 T16 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 25072 T16 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 25097 T16 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 25100 T16 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 25126 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 25139 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 25144 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 25146 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 25146 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 25147 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 25148 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 25148 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 25149 T16 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 25150 T16 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/
   [junit4]   2> 25150 T16 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 25151 T16 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index/
   [junit4]   2> 25155 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 25156 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 25156 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 25157 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 25157 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 25158 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 25158 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 25159 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 25159 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 25160 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 25161 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 25161 T16 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 25192 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25194 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25196 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25199 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25202 T16 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 25204 T16 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json,/replication,spellCheckCompRH_Direct,/get,mock,/search-facet-invariants,mltrh,spellCheckWithWordbreak_Direct,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,dismax,/search-facet-def,spellCheckWithWordbreak,defaults,/admin/segments,/admin/threads,/config,tvrh,/admin/properties,standard,/debug/dump,/update/json/docs,lazy,/mlt,spellCheckCompRH1,/schema,/terms,spellCheckCompRH,/admin/system,/admin/logging,/update,/admin/luke
   [junit4]   2> 25206 T16 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 25209 T16 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 25217 T16 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 25218 T16 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 25219 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5ccd6f35[collection1] main
   [junit4]   2> 25219 T16 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 25220 T16 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 25220 T16 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 25221 T16 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 25224 T16 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 25239 T16 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 25240 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 25240 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 25241 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 25241 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 25241 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 25242 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 25243 T16 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=76.8359375, floorSegmentMB=1.505859375, forceMergeDeletesPctAllowed=23.071419198270572, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 25243 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 25244 T16 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_1,generation=1}
   [junit4]   2> 25245 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 25245 T16 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 25246 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 25246 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 25246 T16 c:collection1 oasc.SolrCore.getSearcher WARN [collection1] PERFORMANCE WARNING: Overlapping onDeckSearchers=2
   [junit4]   2> 25247 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 25247 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@760d62e0[collection1] main
   [junit4]   2> 25248 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 25248 T16 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 25249 T16 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@744e4cf6
   [junit4]   2> 25250 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 25250 T16 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 25250 T20 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5ccd6f35[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 25251 T16 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 25251 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 25253 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 25253 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 25253 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 25255 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 25256 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 25257 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 25258 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 25259 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 25260 T20 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@760d62e0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> ASYNC  NEW_CORE C0 name=collection1 org.apache.solr.core.SolrCore@58169f1b
   [junit4]   2> 26863 T16 c:collection1 C0 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1497736543998050304)]} 0 1594
   [junit4]   2> 27353 T22 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 27945 T16 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SoftAutoCommitTest -Dtests.method=testSoftAndHardCommitMaxTimeMixedAdds -Dtests.seed=E74DB75054A0944 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sr_BA_#Latn -Dtests.timezone=America/Merida -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 3.92s J2 | SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: soft529 wasn't fast enough
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:5FA022F5B43939E3]:0)
   [junit4]    > 	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 28025 T16 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 28026 T16 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 28027 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 28028 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 28029 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 28053 T21 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 28187 T16 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 28231 T16 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 28350 T16 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 28353 T16 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 28361 T22 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6ec3e67f[collection1] main
   [junit4]   2> 28410 T22 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 28411 T20 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6ec3e67f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C1)))}
   [junit4]   2> 28413 T21 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_2,generation=2}
   [junit4]   2> 28413 T21 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 28415 T21 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 28416 T21 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 28513 T16 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 28954 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 28979 T16 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 28982 T16 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 29008 T16 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 29014 T16 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 29019 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 29020 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 29021 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 29021 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 29022 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 29022 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 29023 T16 c:collection1 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 29024 T16 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/
   [junit4]   2> 29024 T16 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 29025 T16 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index/
   [junit4]   2> 29028 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 29029 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 29029 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 29030 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 29030 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 29031 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 29032 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 29033 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 29033 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 29034 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 29035 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 29035 T16 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 29060 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29064 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29066 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29069 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29073 T16 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 29075 T16 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json,/replication,spellCheckCompRH_Direct,/get,mock,/search-facet-invariants,mltrh,spellCheckWithWordbreak_Direct,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,dismax,/search-facet-def,spellCheckWithWordbreak,defaults,/admin/segments,/admin/threads,/config,tvrh,/admin/properties,standard,/debug/dump,/update/json/docs,lazy,/mlt,spellCheckCompRH1,/schema,/terms,spellCheckCompRH,/admin/system,/admin/logging,/update,/admin/luke
   [junit4]   2> 29078 T16 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 29080 T16 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 29082 T16 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 29082 T16 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 29084 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4b3fd89[collection1] main
   [junit4]   2> 29085 T16 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 29086 T16 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 29087 T16 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 29088 T16 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 29092 T16 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 29111 T16 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 29112 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 29112 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 29113 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 29114 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 29114 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 29114 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 29115 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 29115 T16 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 29116 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 29118 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 29119 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 29120 T16 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_2,generation=2}
   [junit4]   2> 29120 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 29120 T16 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 29121 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 29121 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 29122 T16 c:collection1 oasc.SolrCore.getSearcher WARN [collection1] PERFORMANCE WARNING: Overlapping onDeckSearchers=2
   [junit4]   2> 29123 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4b3fd89[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C1)))}
   [junit4]   2> 29128 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7f3b346b[collection1] main
   [junit4]   2> 29130 T16 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 29130 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7f3b346b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C1)))}
   [junit4]   2> 29130 T16 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@58169f1b
   [junit4]   2> 29132 T16 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=1,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=143,transaction_logs_total_number=1}
   [junit4]   2> 29133 T16 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C1 name=collection1 org.apache.solr.core.SolrCore@1073c45f
   [junit4]   2> 29141 T16 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1497736547077718016)]} 0 6
   [junit4]   2> 29144 T16 c:collection1 C1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 29340 T16 c:collection1 C1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_3,generation=3}
   [junit4]   2> 29341 T16 c:collection1 C1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 29342 T16 c:collection1 C1 oass.SolrIndexSearcher.<init> Opening Searcher@61563c31[collection1] main
   [junit4]   2> 29343 T16 c:collection1 C1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 29344 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@61563c31[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(5.2.0):C1)))}
   [junit4]   2> 29346 T16 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 203
   [junit4]   2> 29356 T16 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {delete=[529 (-1497736547307356160)]} 0 3
   [junit4]   2> 29858 T25 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 29868 T25 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4e22d17[collection1] main
   [junit4]   2> 29869 T25 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 29869 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4e22d17[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 29873 T16 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[550 (1497736547848421376)]} 0 3
   [junit4]   2> 30374 T25 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 30389 T25 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@71794a83[collection1] main
   [junit4]   2> 30389 T25 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 30390 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@71794a83[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(5.2.0):C1)))}
   [junit4]   2> 30558 T24 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 30560 T24 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_4,generation=4}
   [junit4]   2> 30560 T24 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 30561 T24 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 30562 T24 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 32562 T16 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 32571 T16 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 32572 T16 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 32573 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 32574 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 32574 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 32684 T16 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 32714 T16 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 32827 T16 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 32828 T16 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 32868 T16 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 33504 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 33535 T16 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 33540 T16 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 33574 T16 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 33587 T16 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 33594 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 33596 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 33596 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 33597 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 33598 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 33599 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 33600 T16 c:collection1 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 33601 T16 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/
   [junit4]   2> 33601 T16 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 33602 T16 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index/
   [junit4]   2> 33607 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 33608 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 33608 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 33609 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 33610 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 33611 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 33611 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 33612 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 33613 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 33614 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 33615 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 33617 T16 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 33640 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33643 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33646 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33649 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33654 T16 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 33657 T16 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json,/replication,spellCheckCompRH_Direct,/get,mock,/search-facet-invariants,mltrh,spellCheckWithWordbreak_Direct,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,dismax,/search-facet-def,spellCheckWithWordbreak,defaults,/admin/segments,/admin/threads,/config,tvrh,/admin/properties,standard,/debug/dump,/update/json/docs,lazy,/mlt,spellCheckCompRH1,/schema,/terms,spellCheckCompRH,/admin/system,/admin/logging,/update,/admin/luke
   [junit4]   2> 33660 T16 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 33662 T16 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 33663 T16 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 33664 T16 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 33666 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1879cd46[collection1] main
   [junit4]   2> 33667 T16 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 33668 T16 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 33668 T16 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 33669 T16 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 33694 T16 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 33709 T16 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 33711 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 33711 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 33712 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 33712 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 33713 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 33713 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 33714 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 33716 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 33717 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 33719 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 33720 T16 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@28a0eaa9
   [junit4]   2> 33721 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 33723 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 33723 T26 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1879cd46[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(5.2.0):C1)))}
   [junit4]   2> 33727 T16 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_4,generation=4}
   [junit4]   2> 33728 T16 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 33729 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 33739 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7cc8ff30[collection1] main
   [junit4]   2> 33741 T16 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 33741 T26 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7cc8ff30[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(5.2.0):C1)))}
   [junit4]   2> 33742 T16 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1073c45f
   [junit4]   2> 33745 T16 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=4,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=2,optimizes=0,

[...truncated too long message...]

1 c:collection1 oasc.SocketProxy.close WARN Closing 2 connections to: http://127.0.0.1:63890/brdco/r
   [junit4]   2> 1504344 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43349/brdco/r
   [junit4]   2> 1504344 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:41051/brdco/r
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43336/brdco/r
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63890/brdco/r
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43349/brdco/r
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:41051/brdco/r
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=E74DB75054A0944 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=nl_NL -Dtests.timezone=Antarctica/McMurdo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   78.9s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:8620E4AFABB664BC]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1504360 T2393 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E74DB75054A0944-001
   [junit4]   2> 78855 T2392 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {range_facet_l=Lucene50(blocksize=128), multiDefault=Lucene50(blocksize=128), timestamp=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), intDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval), _version_=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=LuceneFixedGap), text=PostingsFormat(name=Memory doPackFST= true)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=nl_NL, timezone=Antarctica/McMurdo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=140790216,total=342360064
   [junit4]   2> NOTE: All tests run in this JVM: [EnumFieldTest, TestFaceting, DisMaxRequestHandlerTest, TestSolr4Spatial, TermVectorComponentTest, ExternalFileFieldSortTest, CircularListTest, OpenExchangeRatesOrgProviderTest, RecoveryAfterSoftCommitTest, MultiTermTest, LukeRequestHandlerTest, SaslZkACLProviderTest, JsonLoaderTest, StatelessScriptUpdateProcessorFactoryTest, HdfsUnloadDistributedZkTest, RequiredFieldsTest, TestPostingsSolrHighlighter, RemoteQueryErrorTest, DistributedDebugComponentTest, TestSchemaSimilarityResource, HdfsDirectoryTest, CoreMergeIndexesAdminHandlerTest, DocExpirationUpdateProcessorFactoryTest, TestLMDirichletSimilarityFactory, ShardRoutingCustomTest, TestUpdate, SuggestComponentTest, TestAddFieldRealTimeGet, TestSolrQueryParserDefaultOperatorResource, JSONWriterTest, MultiThreadedOCPTest, TestFuzzyAnalyzedSuggestions, TestClassNameShortening, UUIDUpdateProcessorFallbackTest, TestRebalanceLeaders, SpellCheckComponentTest, TestReload, TestExpandComponent, TestSchemaResource, TestExactSharedStatsCache, CSVRequestHandlerTest, TestAnalyzedSuggestions, TestHashPartitioner, IndexSchemaTest, BasicFunctionalityTest, LoggingHandlerTest, TestStressVersions, IndexBasedSpellCheckerTest, TestNoOpRegenerator, TestSolrJ, PreAnalyzedUpdateProcessorTest, SpellCheckCollatorTest, TestCodecSupport, TestBlobHandler, TestCloudManagedSchema, TestFieldResource, TestIntervalFaceting, BadIndexSchemaTest, LeaderFailoverAfterPartitionTest, TestCloudSchemaless, ReplicationFactorTest]
   [junit4] Completed on J0 in 79.38s, 1 test, 1 error <<< FAILURES!

[...truncated 727 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 53 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 223598798 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2900 - Still Failing

Posted by Mark Miller <ma...@gmail.com>.
Thats a bad fail. Something has probably changed in recovery code.

 - Mark

On Mon, Apr 6, 2015 at 2:11 PM Apache Jenkins Server <
jenkins@builds.apache.org> wrote:

> Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2900/
>
> 4 tests failed.
> REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test
>
> Error Message:
> shard1 is not consistent.  Got 1023 from http://127.0.0.1:10050/p_/z/
> collection1lastClient and got 295 from http://127.0.0.1:10059/p_/z/
> collection1
>
> Stack Trace:
> java.lang.AssertionError: shard1 is not consistent.  Got 1023 from
> http://127.0.0.1:10050/p_/z/collection1lastClient and got 295 from
> http://127.0.0.1:10059/p_/z/collection1
>         at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:
> 73AFD33F7B56D7D1]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.apache.solr.cloud.RecoveryZkTest.test(
> RecoveryZkTest.java:123)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:886)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(
> RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(
> RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:792)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test
>
> Error Message:
> IOException occured when talking to server at:
> http://127.0.0.1:52540/fmq/gz/c8n_1x3_commits_shard1_replica1
>
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:52540/fmq/gz/
> c8n_1x3_commits_shard1_replica1
>         at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:
> 73AFD33F7B56D7D1]:0)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:570)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:233)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:225)
>         at org.apache.solr.client.solrj.SolrRequest.process(
> SolrRequest.java:135)
>         at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.
> java:483)
>         at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.
> java:464)
>         at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommi
> tTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
>         at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(
> LeaderInitiatedRecoveryOnCommitTest.java:64)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:886)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(
> RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(
> RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:792)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:52540
> failed to respond
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:143)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:57)
>         at org.apache.http.impl.io.AbstractMessageParser.parse(
> AbstractMessageParser.java:261)
>         at org.apache.http.impl.AbstractHttpClientConnection.
> receiveResponseHeader(AbstractHttpClientConnection.java:283)
>         at org.apache.http.impl.conn.DefaultClientConnection.
> receiveResponseHeader(DefaultClientConnection.java:251)
>         at org.apache.http.impl.conn.ManagedClientConnectionImpl.
> receiveResponseHeader(ManagedClientConnectionImpl.java:197)
>         at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(
> HttpRequestExecutor.java:272)
>         at org.apache.http.protocol.HttpRequestExecutor.execute(
> HttpRequestExecutor.java:124)
>         at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(
> DefaultRequestDirector.java:685)
>         at org.apache.http.impl.client.DefaultRequestDirector.execute(
> DefaultRequestDirector.java:487)
>         at org.apache.http.impl.client.AbstractHttpClient.doExecute(
> AbstractHttpClient.java:882)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:82)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:107)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:55)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:462)
>         ... 48 more
>
>
> FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test
>
> Error Message:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:50643/collection1
>
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:50643/collection1
>         at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:
> 73AFD33F7B56D7D1]:0)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> directUpdate(CloudSolrClient.java:625)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> sendRequest(CloudSolrClient.java:948)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> requestWithRetryOnStaleState(CloudSolrClient.java:839)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(
> CloudSolrClient.java:782)
>         at org.apache.solr.client.solrj.SolrRequest.process(
> SolrRequest.java:135)
>         at org.apache.solr.client.solrj.SolrClient.add(SolrClient.
> java:174)
>         at org.apache.solr.client.solrj.SolrClient.add(SolrClient.
> java:139)
>         at org.apache.solr.client.solrj.SolrClient.add(SolrClient.
> java:153)
>         at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(
> RecoveryAfterSoftCommitTest.java:88)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:886)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(
> RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(
> RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:792)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.solr.client.solrj.SolrServerException: IOException
> occured when talking to server at: http://127.0.0.1:50643/collection1
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:570)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:233)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:225)
>         at org.apache.solr.client.solrj.impl.LBHttpSolrClient.
> doRequest(LBHttpSolrClient.java:370)
>         at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(
> LBHttpSolrClient.java:325)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> directUpdate(CloudSolrClient.java:622)
>         ... 49 more
> Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:50643
> failed to respond
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:143)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:57)
>         at org.apache.http.impl.io.AbstractMessageParser.parse(
> AbstractMessageParser.java:261)
>         at org.apache.http.impl.AbstractHttpClientConnection.
> receiveResponseHeader(AbstractHttpClientConnection.java:283)
>         at org.apache.http.impl.conn.DefaultClientConnection.
> receiveResponseHeader(DefaultClientConnection.java:251)
>         at org.apache.http.impl.conn.ManagedClientConnectionImpl.
> receiveResponseHeader(ManagedClientConnectionImpl.java:197)
>         at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(
> HttpRequestExecutor.java:272)
>         at org.apache.http.protocol.HttpRequestExecutor.execute(
> HttpRequestExecutor.java:124)
>         at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(
> DefaultRequestDirector.java:685)
>         at org.apache.http.impl.client.DefaultRequestDirector.execute(
> DefaultRequestDirector.java:487)
>         at org.apache.http.impl.client.AbstractHttpClient.doExecute(
> AbstractHttpClient.java:882)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:82)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:107)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:55)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:462)
>         ... 54 more
>
>
> FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test
>
> Error Message:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:11223/
> repfacttest_c8n_1x3_shard1_replica2
>
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:11223/
> repfacttest_c8n_1x3_shard1_replica2
>         at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:
> 73AFD33F7B56D7D1]:0)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> directUpdate(CloudSolrClient.java:625)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> sendRequest(CloudSolrClient.java:948)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> requestWithRetryOnStaleState(CloudSolrClient.java:839)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(
> CloudSolrClient.java:782)
>         at org.apache.solr.client.solrj.SolrClient.request(SolrClient.
> java:1220)
>         at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(
> ReplicationFactorTest.java:308)
>         at org.apache.solr.cloud.ReplicationFactorTest.testRf3(
> ReplicationFactorTest.java:234)
>         at org.apache.solr.cloud.ReplicationFactorTest.test(
> ReplicationFactorTest.java:113)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:886)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(
> RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(
> RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:792)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.solr.client.solrj.SolrServerException: IOException
> occured when talking to server at: http://127.0.0.1:11223/
> repfacttest_c8n_1x3_shard1_replica2
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:570)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:233)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:225)
>         at org.apache.solr.client.solrj.impl.LBHttpSolrClient.
> doRequest(LBHttpSolrClient.java:370)
>         at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(
> LBHttpSolrClient.java:325)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> directUpdate(CloudSolrClient.java:622)
>         ... 48 more
> Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:11223
> failed to respond
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:143)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:57)
>         at org.apache.http.impl.io.AbstractMessageParser.parse(
> AbstractMessageParser.java:261)
>         at org.apache.http.impl.AbstractHttpClientConnection.
> receiveResponseHeader(AbstractHttpClientConnection.java:283)
>         at org.apache.http.impl.conn.DefaultClientConnection.
> receiveResponseHeader(DefaultClientConnection.java:251)
>         at org.apache.http.impl.conn.ManagedClientConnectionImpl.
> receiveResponseHeader(ManagedClientConnectionImpl.java:197)
>         at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(
> HttpRequestExecutor.java:272)
>         at org.apache.http.protocol.HttpRequestExecutor.execute(
> HttpRequestExecutor.java:124)
>         at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(
> DefaultRequestDirector.java:685)
>         at org.apache.http.impl.client.DefaultRequestDirector.execute(
> DefaultRequestDirector.java:487)
>         at org.apache.http.impl.client.AbstractHttpClient.doExecute(
> AbstractHttpClient.java:882)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:82)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:107)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:55)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:462)
>         ... 53 more
>
>
>
>
> Build Log:
> [...truncated 9494 lines...]
>    [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
>    [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/init-core-data-001
>    [junit4]   2> 295434 T1269 oas.BaseDistributedSearchTestCase.initHostContext
> Setting hostContext system property: /
>    [junit4]   2> 295441 T1269 oasc.ZkTestServer.run STARTING ZK TEST SERVER
>    [junit4]   1> client port:0.0.0.0/0.0.0.0:0
>    [junit4]   2> 295441 T1270 oasc.ZkTestServer$ZKServerMain.runFromConfig
> Starting server
>    [junit4]   2> 295541 T1269 oasc.ZkTestServer.run start zk server on
> port:50623
>    [junit4]   2> 295542 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 295542 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 295545 T1277 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@6e5605f4
> name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent
> state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 295545 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 295546 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 295546 T1269 oascc.SolrZkClient.makePath makePath: /solr
>    [junit4]   2> 295548 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 295549 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 295550 T1280 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@842caa6
> name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 295550 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 295551 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 295551 T1269 oascc.SolrZkClient.makePath makePath:
> /collections/collection1
>    [junit4]   2> 295553 T1269 oascc.SolrZkClient.makePath makePath:
> /collections/collection1/shards
>    [junit4]   2> 295554 T1269 oascc.SolrZkClient.makePath makePath:
> /collections/control_collection
>    [junit4]   2> 295555 T1269 oascc.SolrZkClient.makePath makePath:
> /collections/control_collection/shards
>    [junit4]   2> 295556 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
> to /configs/conf1/solrconfig.xml
>    [junit4]   2> 295557 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/solrconfig.xml
>    [junit4]   2> 295559 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
> to /configs/conf1/schema.xml
>    [junit4]   2> 295560 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/schema.xml
>    [junit4]   2> 295661 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
> conf/solrconfig.snippet.randomindexconfig.xml to
> /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>    [junit4]   2> 295662 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>    [junit4]   2> 295663 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
> to /configs/conf1/stopwords.txt
>    [junit4]   2> 295663 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/stopwords.txt
>    [junit4]   2> 295665 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt
> to /configs/conf1/protwords.txt
>    [junit4]   2> 295665 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/protwords.txt
>    [junit4]   2> 295667 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml
> to /configs/conf1/currency.xml
>    [junit4]   2> 295667 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/currency.xml
>    [junit4]   2> 295668 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
> to /configs/conf1/enumsConfig.xml
>    [junit4]   2> 295669 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/enumsConfig.xml
>    [junit4]   2> 295670 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
> to /configs/conf1/open-exchange-rates.json
>    [junit4]   2> 295670 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/open-exchange-rates.json
>    [junit4]   2> 295672 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
> conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-
> ISOLatin1Accent.txt
>    [junit4]   2> 295672 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/mapping-ISOLatin1Accent.txt
>    [junit4]   2> 295673 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
> to /configs/conf1/old_synonyms.txt
>    [junit4]   2> 295674 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/old_synonyms.txt
>    [junit4]   2> 295675 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
> to /configs/conf1/synonyms.txt
>    [junit4]   2> 295675 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/synonyms.txt
>    [junit4]   2> 296963 T1269 oas.SolrTestCaseJ4.writeCoreProperties
> Writing core.properties file to /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1
>    [junit4]   2> 296967 T1269 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 296971 T1269 oejs.AbstractConnector.doStart Started
> SocketConnector@127.0.0.1:50636
>    [junit4]   2> 296971 T1269 oascse.JettySolrRunner$1.lifeCycleStarted
> Jetty properties: {hostPort=50635, coreRootDirectory=/usr/home/
> jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.
> x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores, hostContext=/,
> solr.data.dir=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/tempDir-001/control/data}
>    [junit4]   2> 296971 T1269 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
>    [junit4]   2> 296972 T1269 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/'
>    [junit4]   2> 297015 T1269 oasc.SolrXmlConfig.fromFile Loading
> container configuration from /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/solr.xml
>    [junit4]   2> 297067 T1269 oasc.CorePropertiesLocator.<init>
> Config-defined core root directory: /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores
>    [junit4]   2> 297069 T1269 oasc.CoreContainer.<init> New CoreContainer
> 2110879613
>    [junit4]   2> 297069 T1269 oasc.CoreContainer.load Loading cores into
> CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/]
>    [junit4]   2> 297069 T1269 oasc.CoreContainer.load loading shared
> library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/lib
>    [junit4]   2> 297070 T1269 oasc.SolrResourceLoader.addToClassLoader
> WARN Can't find (or read) directory to add to classloader: lib (resolved
> as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/lib).
>    [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 90000
>    [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to:
>    [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>    [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>    [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnections to: 10000
>    [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>    [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>    [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>    [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>    [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>    [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting useRetries to: false
>    [junit4]   2> 297089 T1269 oasu.UpdateShardHandler.<init> Creating
> UpdateShardHandler HTTP client with params: socketTimeout=340000&
> connTimeout=45000&retry=true
>    [junit4]   2> 297090 T1269 oasl.LogWatcher.createWatcher SLF4J impl is
> org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 297090 T1269 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 297090 T1269 oasc.CoreContainer.load Node Name: 127.0.0.1
>    [junit4]   2> 297091 T1269 oasc.ZkContainer.initZooKeeper Zookeeper
> client=127.0.0.1:50623/solr
>    [junit4]   2> 297091 T1269 oasc.ZkController.checkChrootPath zkHost
> includes chroot
>    [junit4]   2> 297092 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 297092 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 297094 T1292 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@45cc06e8
> name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent
> state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 297095 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 297096 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 297097 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 297098 T1295 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@34504d13
> name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 297098 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 297100 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/queue
>    [junit4]   2> 297101 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-queue-work
>    [junit4]   2> 297103 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-map-running
>    [junit4]   2> 297104 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-map-completed
>    [junit4]   2> 297105 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-map-failure
>    [junit4]   2> 297107 T1269 oascc.SolrZkClient.makePath makePath:
> /live_nodes
>    [junit4]   2> 297108 T1269 oascc.SolrZkClient.makePath makePath:
> /aliases.json
>    [junit4]   2> 297109 T1269 oascc.SolrZkClient.makePath makePath:
> /clusterstate.json
>    [junit4]   2> 297110 T1269 oasc.ZkController.createEphemeralLiveNode
> Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50635_
>    [junit4]   2> 297110 T1269 oascc.SolrZkClient.makePath makePath:
> /live_nodes/127.0.0.1:50635_
>    [junit4]   2> 297112 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer_elect
>    [junit4]   2> 297113 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer_elect/election
>    [junit4]   2> 297114 T1269 oasc.Overseer.close Overseer (id=null)
> closing
>    [junit4]   2> 297115 T1269 oasc.OverseerElectionContext.runLeaderProcess
> I am going to be the leader 127.0.0.1:50635_
>    [junit4]   2> 297115 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer_elect/leader
>    [junit4]   2> 297116 T1269 oasc.Overseer.start Overseer
> (id=93607700461191171-127.0.0.1:50635_-n_0000000000) starting
>    [junit4]   2> 297118 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/queue-work
>    [junit4]   2> 297122 T1269 oasc.OverseerAutoReplicaFailoverThread.<init>
> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000
> autoReplicaFailoverWaitAfterExpiration=30000
> autoReplicaFailoverBadNodeExpiration=60000
>    [junit4]   2> 297122 T1297 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 297122 T1269 oascc.ZkStateReader.
> createClusterStateWatchersAndUpdate Updating cluster state from
> ZooKeeper...
>    [junit4]   2> 297122 T1296 oasc.Overseer$ClusterStateUpdater.run
> Starting to work on the main queue
>    [junit4]   2> 297126 T1269 oasc.CorePropertiesLocator.discover Looking
> for core definitions underneath /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores
>    [junit4]   2> 297128 T1269 oasc.CoreDescriptor.<init> CORE DESCRIPTOR:
> {shard=, collection=control_collection, coreNodeName=, dataDir=data/,
> absoluteInstDir=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/,
> config=solrconfig.xml, schema=schema.xml, loadOnStartup=true,
> transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/
> Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1, name=collection1}
>    [junit4]   2> 297128 T1269 oasc.CorePropertiesLocator.discoverUnder
> Found core collection1 in /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/
>    [junit4]   2> 297128 T1269 oasc.CorePropertiesLocator.discover Found 1
> core definitions
>    [junit4]   2> 297130 T1299 C:control_collection c:collection1
> oasc.ZkController.publish publishing core=collection1 state=down
> collection=control_collection
>    [junit4]   2> 297130 T1299 C:control_collection c:collection1
> oasc.ZkController.publish numShards not found on descriptor - reading it
> from system property
>    [junit4]   2> 297131 T1299 C:control_collection c:collection1
> oasc.ZkController.waitForCoreNodeName look for our core node name
>    [junit4]   2> 297131 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 297132 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:50635_",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "numShards":"1"} current state version: 0
>    [junit4]   2> 297132 T1296 oasco.ReplicaMutator.updateState Update
> state numShards=1 message={
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:50635_",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "numShards":"1"}
>    [junit4]   2> 297132 T1296 oasco.ClusterStateMutator.createCollection
> building a new cName: control_collection
>    [junit4]   2> 297133 T1296 oasco.ReplicaMutator.updateState Assigning
> new node to shard shard=shard1
>    [junit4]   2> 297134 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298131 T1299 C:control_collection c:collection1
> oasc.ZkController.waitForShardId waiting to find shard id in clusterstate
> for collection1
>    [junit4]   2> 298132 T1299 C:control_collection c:collection1
> oasc.ZkController.createCollectionZkNode Check for collection
> zkNode:control_collection
>    [junit4]   2> 298132 T1299 C:control_collection c:collection1
> oasc.ZkController.createCollectionZkNode Collection zkNode exists
>    [junit4]   2> 298132 T1299 C:control_collection c:collection1
> oascc.ZkStateReader.readConfigName Load collection config
> from:/collections/control_collection
>    [junit4]   2> 298133 T1299 C:control_collection c:collection1
> oascc.ZkStateReader.readConfigName path=/collections/control_collection
> configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 298133 T1299 C:control_collection c:collection1
> oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory:
> '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/'
>    [junit4]   2> 298147 T1299 C:control_collection c:collection1
> oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
>    [junit4]   2> 298149 T1299 C:control_collection c:collection1
> oasc.Config.<init> loaded config solrconfig.xml with version 0
>    [junit4]   2> 298156 T1299 C:control_collection c:collection1
> oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
>    [junit4]   2> 298172 T1299 C:control_collection c:collection1
> oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
>    [junit4]   2> 298210 T1299 C:control_collection c:collection1
> oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>    [junit4]   2> 298311 T1299 C:control_collection c:collection1
> oass.IndexSchema.readSchema Reading Solr Schema from
> /configs/conf1/schema.xml
>    [junit4]   2> 298318 T1299 C:control_collection c:collection1
> oass.IndexSchema.readSchema [collection1] Schema name=test
>    [junit4]   2> 298623 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider.init Initialized with
> rates=open-exchange-rates.json, refreshInterval=1440.
>    [junit4]   2> 298636 T1299 C:control_collection c:collection1
> oass.IndexSchema.readSchema default search field in schema is text
>    [junit4]   2> 298638 T1299 C:control_collection c:collection1
> oass.IndexSchema.readSchema unique key field: id
>    [junit4]   2> 298648 T1299 C:control_collection c:collection1
> oass.FileExchangeRateProvider.reload Reloading exchange rates from file
> currency.xml
>    [junit4]   2> 298651 T1299 C:control_collection c:collection1
> oass.FileExchangeRateProvider.reload Reloading exchange rates from file
> currency.xml
>    [junit4]   2> 298654 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider.reload Reloading exchange rates from
> open-exchange-rates.json
>    [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
> IMPORTANT NOTE
>    [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key,
> got STRING
>    [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider.reload Reloading exchange rates from
> open-exchange-rates.json
>    [junit4]   2> 298656 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
> IMPORTANT NOTE
>    [junit4]   2> 298656 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key,
> got STRING
>    [junit4]   2> 298656 T1299 C:control_collection c:collection1
> oasc.CoreContainer.create Creating SolrCore 'collection1' using
> configuration from collection control_collection
>    [junit4]   2> 298656 T1299 C:control_collection c:collection1
> oasc.SolrCore.initDirectoryFactory org.apache.solr.core.
> MockDirectoryFactory
>    [junit4]   2> 298657 T1299 C:control_collection c:collection1
> oasc.SolrCore.<init> [collection1] Opening new SolrCore at
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/,
> dataDir=/usr/home/jenkins/jenkins-slave/workspace/
> Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data/
>    [junit4]   2> 298657 T1299 C:control_collection c:collection1
> oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans
> to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ec2b6ef
>    [junit4]   2> 298658 T1299 C:control_collection c:collection1
> oasc.CachingDirectoryFactory.get return new directory for
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data
>    [junit4]   2> 298658 T1299 C:control_collection c:collection1
> oasc.SolrCore.getNewIndexDir New index directory detected: old=null
> new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index/
>    [junit4]   2> 298658 T1299 C:control_collection c:collection1
> oasc.SolrCore.initIndex WARN [collection1] Solr index directory
> '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index' doesn't
> exist. Creating new index...
>    [junit4]   2> 298658 T1299 C:control_collection c:collection1
> oasc.CachingDirectoryFactory.get return new directory for
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index
>    [junit4]   2> 298659 T1299 C:control_collection c:collection1
> oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class
> org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
> maxMergeAtOnce=27, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=52.0615234375,
> floorSegmentMB=0.486328125, forceMergeDeletesPctAllowed=10.112383767408401,
> segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12,
> noCFSRatio=1.0
>    [junit4]   2> 298659 T1299 C:control_collection c:collection1
> oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits:
> num=1
>    [junit4]   2>                commit{dir=MockDirectoryWrapper(
> RAMDirectory@6041646a lockFactory=org.apache.lucene.store.
> SingleInstanceLockFactory@65279c84),segFN=segments_1,generation=1}
>    [junit4]   2> 298659 T1299 C:control_collection c:collection1
> oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "nodistrib"
>    [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "dedupe"
>    [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory
> into updateRequestProcessorChain "dedupe"
>    [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "stored_sig"
>    [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory
> into updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "distrib-dup-test-chain-explicit"
>    [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "distrib-dup-test-chain-implicit"
>    [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory
> into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 298666 T1299 C:control_collection c:collection1
> oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain
> defined as default, creating implicit default
>    [junit4]   2> 298685 T1299 C:control_collection c:collection1
> oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 298687 T1299 C:control_collection c:collection1
> oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 298689 T1299 C:control_collection c:collection1
> oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 298690 T1299 C:control_collection c:collection1
> oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 298694 T1299 C:control_collection c:collection1
> oasc.RequestHandlers.initHandlersFromConfig Registered paths:
> /schema,/update/json/docs,/admin/threads,/config,/
> replication,/admin/file,/admin/logging,/admin/plugins,/
> update/csv,standard,/admin/ping,/admin/system,/get,/
> admin/luke,/admin/segments,/admin/mbeans,/admin/
> properties,/update,/update/json
>    [junit4]   2> 298696 T1299 C:control_collection c:collection1
> oasc.SolrCore.initStatsCache Using default statsCache cache:
> org.apache.solr.search.stats.LocalStatsCache
>    [junit4]   2> 298697 T1299 C:control_collection c:collection1
> oasu.UpdateHandler.<init> Using UpdateLog implementation:
> org.apache.solr.update.UpdateLog
>    [junit4]   2> 298697 T1299 C:control_collection c:collection1
> oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH
> numRecordsToKeep=100 maxNumLogsToKeep=10
>    [junit4]   2> 298699 T1299 C:control_collection c:collection1
> oasu.CommitTracker.<init> Hard AutoCommit: disabled
>    [junit4]   2> 298699 T1299 C:control_collection c:collection1
> oasu.CommitTracker.<init> Soft AutoCommit: disabled
>    [junit4]   2> 298699 T1299 C:control_collection c:collection1
> oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class
> org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
> maxMergeAtOnce=30, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=39.86328125,
> floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=25.115751995354497,
> segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12,
> noCFSRatio=0.0
>    [junit4]   2> 298700 T1299 C:control_collection c:collection1
> oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>                commit{dir=MockDirectoryWrapper(
> RAMDirectory@6041646a lockFactory=org.apache.lucene.store.
> SingleInstanceLockFactory@65279c84),segFN=segments_1,generation=1}
>    [junit4]   2> 298700 T1299 C:control_collection c:collection1
> oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 298700 T1299 C:control_collection c:collection1
> oass.SolrIndexSearcher.<init> Opening Searcher@445242be[collection1] main
>    [junit4]   2> 298701 T1299 C:control_collection c:collection1
> oascc.ZkStateReader.readConfigName Load collection config
> from:/collections/control_collection
>    [junit4]   2> 298701 T1299 C:control_collection c:collection1
> oascc.ZkStateReader.readConfigName path=/collections/control_collection
> configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 298701 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based
> storage for the RestManager with znodeBase: /configs/conf1
>    [junit4]   2> 298702 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured
> ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 298702 T1299 C:control_collection c:collection1
> oasr.RestManager.init Initializing RestManager with initArgs: {}
>    [junit4]   2> 298702 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage.load Reading _rest_managed.json using
> ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 298703 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data
> found for znode /configs/conf1/_rest_managed.json
>    [junit4]   2> 298703 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json
> using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 298703 T1299 C:control_collection c:collection1
> oasr.RestManager.init Initializing 0 registered ManagedResources
>    [junit4]   2> 298703 T1299 C:control_collection c:collection1
> oash.ReplicationHandler.inform Commits will be reserved for  10000
>    [junit4]   2> 298704 T1299 C:control_collection c:collection1
> oasc.CoreContainer.registerCore registering core: collection1
>    [junit4]   2> 298704 T1300 C:control_collection c:collection1
> oasc.SolrCore.registerSearcher [collection1] Registered new searcher
> Searcher@445242be[collection1] main{ExitableDirectoryReader(
> UninvertingDirectoryReader())}
>    [junit4]   2> 298704 T1303 C:control_collection S:shard1
> oasc.ZkController.register Register replica - core:collection1 address:
> http://127.0.0.1:50635 collection:control_collection shard:shard1
>    [junit4]   2> 298705 T1269 oass.SolrDispatchFilter.init
> user.dir=/usr/home/jenkins/jenkins-slave/workspace/
> Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
>    [junit4]   2> 298706 T1269 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init() done
>    [junit4]   2> 298706 T1303 C:control_collection S:shard1
> oascc.SolrZkClient.makePath makePath: /collections/control_
> collection/leader_elect/shard1/election
>    [junit4]   2> 298710 T1303 C:control_collection S:shard1 oasc.
> ShardLeaderElectionContext.runLeaderProcess Running the leader process
> for shard shard1
>    [junit4]   2> 298710 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 298711 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 298711 T1303 C:control_collection S:shard1 oasc.
> ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found
> to continue.
>    [junit4]   2> 298711 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 298712 T1303 C:control_collection S:shard1 oasc.
> ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try
> and sync
>    [junit4]   2> ASYNC  NEW_CORE C279 name=collection1
> org.apache.solr.core.SolrCore@1745c55d url=http://127.0.0.1:50635/
> collection1 node=127.0.0.1:50635_ C279_STATE=coll:control_collection
> core:collection1 props:{state=down, base_url=http://127.0.0.1:50635,
> node_name=127.0.0.1:50635_, core=collection1}
>    [junit4]   2> 298712 T1303 C:control_collection S:shard1 C279 P50635
> oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50635/
> collection1/
>    [junit4]   2> 298712 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "operation":"leader",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "collection":"control_collection"} current
> state version: 1
>    [junit4]   2> 298712 T1307 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@6ea67455
> name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 298712 T1303 C:control_collection S:shard1 C279 P50635
> oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
>    [junit4]   2> 298713 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 298713 T1303 C:control_collection S:shard1 C279 P50635
> oasc.SyncStrategy.syncToMe http://127.0.0.1:50635/collection1/ has no
> replicas
>    [junit4]   2> 298713 T1303 C:control_collection S:shard1 oasc.
> ShardLeaderElectionContext.runLeaderProcess I am the new leader:
> http://127.0.0.1:50635/collection1/ shard1
>    [junit4]   2> 298714 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 298714 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298714 T1303 C:control_collection S:shard1
> oascc.SolrZkClient.makePath makePath: /collections/control_
> collection/leaders/shard1
>    [junit4]   2> 298714 T1269 oascc.ZkStateReader.
> createClusterStateWatchersAndUpdate Updating cluster state from
> ZooKeeper...
>    [junit4]   2> 298717 T1269 oasc.ChaosMonkey.monkeyLog monkey: init -
> expire sessions:false cause connection loss:false
>    [junit4]   2> 298718 T1269 oasc.AbstractFullDistribZkTestBase.createJettys
> Creating collection1 with stateFormat=2
>    [junit4]   2> 298719 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 298719 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 298719 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 298720 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "operation":"leader",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "state":"active"} current state version: 2
>    [junit4]   2> 298720 T1310 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@5c566433
> name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 298721 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 298721 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 298722 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 298723 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "operation":"create",
>    [junit4]   2>          "name":"collection1",
>    [junit4]   2>          "numShards":"1",
>    [junit4]   2>          "stateFormat":"2"} current state version: 2
>    [junit4]   2> 298723 T1296 oasco.ClusterStateMutator.createCollection
> building a new cName: collection1
>    [junit4]   2> 298724 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298724 T1307 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298769 T1303 C:control_collection S:shard1
> oasc.ZkController.register We are http://127.0.0.1:50635/collection1/ and
> leader is http://127.0.0.1:50635/collection1/
>    [junit4]   2> 298770 T1303 C:control_collection S:shard1
> oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=
> http://127.0.0.1:50635
>    [junit4]   2> 298770 T1303 C:control_collection S:shard1
> oasc.ZkController.checkRecovery I am the leader, no recovery necessary
>    [junit4]   2> 298770 T1303 C:control_collection S:shard1 c:collection1
> oasc.ZkController.publish publishing core=collection1 state=active
> collection=control_collection
>    [junit4]   2> 298770 T1303 C:control_collection S:shard1 c:collection1
> oasc.ZkController.publish numShards not found on descriptor - reading it
> from system property
>    [junit4]   2> 298771 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 298772 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "core_node_name":"core_node1",
>    [junit4]   2>          "state":"active",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:50635_",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "numShards":"1"} current state version: 3
>    [junit4]   2> 298773 T1296 oasco.ReplicaMutator.updateState Update
> state numShards=1 message={
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "core_node_name":"core_node1",
>    [junit4]   2>          "state":"active",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:50635_",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "numShards":"1"}
>    [junit4]   2> 298773 T1296 oasco.ZkStateWriter.writePendingUpdates
> going to create_collection /collections/collection1/state.json
>    [junit4]   2> 298774 T1307 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298774 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298878 T1307 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298878 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 300125 T1269 oas.SolrTestCaseJ4.writeCoreProperties
> Writing core.properties file to /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/cores/collection1
>    [junit4]   2> 300128 T1269 oasc.AbstractFullDistribZkTestBase.createJettys
> create jetty 1 in directory /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001
>    [junit4]   2> 300128 T1269 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 300132 T1269 oejs.AbstractConnector.doStart Started
> SocketConnector@127.0.0.1:50644
>    [junit4]   2> 300132 T1269 oascse.JettySolrRunner$1.lifeCycleStarted
> Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/cores, solr.data.dir=/usr/home/
> jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.
> x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/tempDir-001/jetty1, hostContext=/,
> solrconfig=solrconfig.xml, hostPort=50643}
>    [junit4]   2> 300133 T1269 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
>    [junit4]   2> 300133 T1269 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/'
>    [junit4]   2> 300171 T1269 oasc.SolrXmlConfig.fromFile Loading
> container configuration from /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/solr.xml
>    [junit4]   2> 300213 T1269 oasc.CorePropertiesLocator.<init>
> Config-defined core root directory: /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/cores
>    [junit4]   2> 300213 T1269 oasc.CoreContainer.<init> New CoreContainer
> 1370575573
>    [junit4]   2> 300214 T1269 oasc.CoreContainer.load Loading cores into
> CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/]
>    [junit4]   2> 300214 T1269 oasc.CoreContainer.load loading shared
> library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/lib
>    [junit4]   2> 300215 T1269 oasc.SolrResourceLoader.addToClassLoader
> WARN Can