You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2014/08/13 05:55:15 UTC

[JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 596 - Still Failing

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/596/

4 tests failed.
FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:25801/_h/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25801/_h/collection1
	at __randomizedtesting.SeedInfo.seed([E7864240DC5F2611:6660CC58AB00462D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:562)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:223)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:165)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrServer(FullSolrCloudDistribCmdsTest.java:414)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.doTest(FullSolrCloudDistribCmdsTest.java:144)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:867)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketException: Broken pipe
	at java.net.SocketOutputStream.socketWrite0(Native Method)
	at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113)
	at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
	at org.apache.http.impl.io.AbstractSessionOutputBuffer.flushBuffer(AbstractSessionOutputBuffer.java:159)
	at org.apache.http.impl.io.AbstractSessionOutputBuffer.writeLine(AbstractSessionOutputBuffer.java:258)
	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:59)
	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:46)
	at org.apache.http.impl.io.AbstractMessageWriter.write(AbstractMessageWriter.java:109)
	at org.apache.http.impl.AbstractHttpClientConnection.sendRequestHeader(AbstractHttpClientConnection.java:254)
	at org.apache.http.impl.conn.DefaultClientConnection.sendRequestHeader(DefaultClientConnection.java:270)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.sendRequestHeader(ManagedClientConnectionImpl.java:209)
	at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:203)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:121)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:448)
	... 50 more


FAILED:  org.apache.solr.request.TestIntervalFaceting.testIntFieldsMultipleSegments

Error Message:
Expected multiple reader leaves. Found 1

Stack Trace:
java.lang.AssertionError: Expected multiple reader leaves. Found 1
	at __randomizedtesting.SeedInfo.seed([E7864240DC5F2611:E225149F98057EC8]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.request.TestIntervalFaceting.assertMultipleReaders(TestIntervalFaceting.java:134)
	at org.apache.solr.request.TestIntervalFaceting.testIntFieldsMultipleSegments(TestIntervalFaceting.java:711)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.request.TestIntervalFaceting.testMultipleSegments

Error Message:
Expected multiple reader leaves. Found 1

Stack Trace:
java.lang.AssertionError: Expected multiple reader leaves. Found 1
	at __randomizedtesting.SeedInfo.seed([E7864240DC5F2611:5BF1F7191BF88A54]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.request.TestIntervalFaceting.assertMultipleReaders(TestIntervalFaceting.java:134)
	at org.apache.solr.request.TestIntervalFaceting.testMultipleSegments(TestIntervalFaceting.java:103)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.request.TestIntervalFaceting.testMultiValueFields

Error Message:
Expected multiple reader leaves. Found 1

Stack Trace:
java.lang.AssertionError: Expected multiple reader leaves. Found 1
	at __randomizedtesting.SeedInfo.seed([E7864240DC5F2611:EE26DEDEBE485AEB]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.request.TestIntervalFaceting.assertMultipleReaders(TestIntervalFaceting.java:134)
	at org.apache.solr.request.TestIntervalFaceting.testMultiValueFields(TestIntervalFaceting.java:71)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11757 lines...]
   [junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/init-core-data-001
   [junit4]   2> 535000 T685 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_h/
   [junit4]   2> 535006 T685 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 535007 T685 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 535008 T686 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 535108 T685 oasc.ZkTestServer.run start zk server on port:25798
   [junit4]   2> 535109 T685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 535115 T692 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3bc4520 name:ZooKeeperConnection Watcher:127.0.0.1:25798 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 535115 T685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 535115 T685 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 535120 T685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 535121 T694 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@189513ae name:ZooKeeperConnection Watcher:127.0.0.1:25798/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 535121 T685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 535121 T685 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 535124 T685 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 535125 T685 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 535126 T685 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 535128 T685 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 535129 T685 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 535131 T685 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 535132 T685 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 535134 T685 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 535134 T685 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 535136 T685 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 535137 T685 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 535138 T685 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 535139 T685 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 535141 T685 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 535141 T685 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 535143 T685 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 535144 T685 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 535145 T685 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 535146 T685 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 535148 T685 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 535148 T685 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 535150 T685 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 535151 T685 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 535152 T685 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 535153 T685 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 536344 T685 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 536349 T685 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25801
   [junit4]   2> 536349 T685 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 536350 T685 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 536350 T685 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002
   [junit4]   2> 536350 T685 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/'
   [junit4]   2> 536380 T685 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/solr.xml
   [junit4]   2> 536430 T685 oasc.CoreContainer.<init> New CoreContainer 64529212
   [junit4]   2> 536430 T685 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/]
   [junit4]   2> 536431 T685 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 536431 T685 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 536431 T685 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 536432 T685 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 536432 T685 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 536432 T685 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 536432 T685 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 536433 T685 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 536433 T685 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 536433 T685 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 536434 T685 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 536434 T685 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 536434 T685 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 536434 T685 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25798/solr
   [junit4]   2> 536435 T685 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 536435 T685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 536437 T705 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3c4ba1d name:ZooKeeperConnection Watcher:127.0.0.1:25798 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 536438 T685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 536441 T685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 536442 T707 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@93cb9b4 name:ZooKeeperConnection Watcher:127.0.0.1:25798/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 536442 T685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 536444 T685 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 536446 T685 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 536448 T685 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 536449 T685 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 536451 T685 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 536453 T685 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 536454 T685 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25801__h
   [junit4]   2> 536455 T685 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25801__h
   [junit4]   2> 536457 T685 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 536458 T685 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 536460 T685 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:25801__h
   [junit4]   2> 536460 T685 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 536462 T685 oasc.Overseer.start Overseer (id=92267852054724611-127.0.0.1:25801__h-n_0000000000) starting
   [junit4]   2> 536463 T685 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 536469 T709 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 536469 T685 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 536471 T685 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 536472 T685 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 536473 T708 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 536478 T710 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 536478 T710 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 536479 T710 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 536479 T711 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 536480 T708 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 536481 T708 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25801/_h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25801__h",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 536481 T708 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 536481 T708 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 536483 T711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 537479 T710 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 537480 T710 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 537480 T710 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 537480 T710 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 537481 T710 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 537481 T710 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/collection1/'
   [junit4]   2> 537483 T710 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 537484 T710 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 537485 T710 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 537548 T710 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 537593 T710 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 537595 T710 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 537604 T710 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 537976 T710 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 537979 T710 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 537981 T710 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 537985 T710 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 538004 T710 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 538005 T710 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 538005 T710 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/collection1/, dataDir=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/control/data/
   [junit4]   2> 538005 T710 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6d87b6ba
   [junit4]   2> 538006 T710 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/control/data
   [junit4]   2> 538006 T710 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/control/data/index/
   [junit4]   2> 538007 T710 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 538008 T710 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/control/data/index
   [junit4]   2> 538008 T710 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 538009 T710 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=BaseDirectoryWrapper(RAMDirectory@33b526df lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1b26d6ff),segFN=segments_1,generation=1}
   [junit4]   2> 538009 T710 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 538014 T710 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 538014 T710 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 538014 T710 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 538015 T710 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 538015 T710 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 538015 T710 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 538016 T710 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 538016 T710 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 538016 T710 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 538017 T710 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 538017 T710 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 538018 T710 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 538018 T710 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 538019 T710 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 538019 T710 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 538020 T710 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 538020 T710 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 538020 T710 oasc.RequestHandlers.initHandlersFromConfig WARN Multiple requestHandler registered to the same name: /update ignoring: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 538021 T710 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 538021 T710 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 538022 T710 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 538040 T710 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 538042 T710 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 538044 T710 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 538046 T710 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 538048 T710 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 538053 T710 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 538054 T710 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 538054 T710 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@1760655f
   [junit4]   2> 538055 T710 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=BaseDirectoryWrapper(RAMDirectory@33b526df lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1b26d6ff),segFN=segments_1,generation=1}
   [junit4]   2> 538055 T710 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 538055 T710 oass.SolrIndexSearcher.<init> Opening Searcher@9342108[collection1] main
   [junit4]   2> 538056 T710 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 538056 T710 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 538057 T710 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 538057 T710 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 538057 T710 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 538058 T710 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 538058 T710 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 538058 T710 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 538059 T710 oascc.SolrZkClient.makePath makePath: /configs/conf1/_rest_managed.json
   [junit4]   2> 538061 T710 oasr.ManagedResourceStorage$ZooKeeperStorageIO$1.close Wrote 38 bytes to new znode /configs/conf1/_rest_managed.json
   [junit4]   2> 538061 T710 oasr.ManagedResourceStorage$JsonStorage.store Saved JSON object to path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 538061 T710 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 538064 T712 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@9342108[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 538068 T710 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 538068 T715 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25801/_h collection:control_collection shard:shard1
   [junit4]   2> 538069 T685 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
   [junit4]   2> 538069 T685 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 538069 T715 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 538070 T685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 538072 T717 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ab4a786 name:ZooKeeperConnection Watcher:127.0.0.1:25798/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 538072 T685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 538073 T685 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 538075 T715 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 538075 T685 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 538076 T711 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 538077 T715 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 538077 T715 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C273 name=collection1 org.apache.solr.core.SolrCore@5fb62a19 url=http://127.0.0.1:25801/_h/collection1 node=127.0.0.1:25801__h C273_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:25801/_h, core=collection1, node_name=127.0.0.1:25801__h}
   [junit4]   2> 538077 T715 C273 P25801 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25801/_h/collection1/
   [junit4]   2> 538077 T708 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 538077 T715 C273 P25801 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 538078 T715 C273 P25801 oasc.SyncStrategy.syncToMe http://127.0.0.1:25801/_h/collection1/ has no replicas
   [junit4]   2> 538078 T715 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25801/_h/collection1/ shard1
   [junit4]   2> 538078 T715 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 538080 T711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 538080 T718 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 538083 T708 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 538085 T711 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 538086 T718 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 538086 T711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 538133 T715 oasc.ZkController.register We are http://127.0.0.1:25801/_h/collection1/ and leader is http://127.0.0.1:25801/_h/collection1/
   [junit4]   2> 538134 T715 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25801/_h
   [junit4]   2> 538134 T715 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 538134 T715 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 538134 T715 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 538136 T711 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 538136 T715 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 538137 T708 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 538137 T708 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25801/_h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25801__h",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 538240 T711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 538240 T718 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 539289 T685 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 539290 T685 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 539293 T685 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25805
   [junit4]   2> 539294 T685 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 539294 T685 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 539294 T685 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003
   [junit4]   2> 539294 T685 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/'
   [junit4]   2> 539323 T685 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/solr.xml
   [junit4]   2> 539384 T685 oasc.CoreContainer.<init> New CoreContainer 1044030314
   [junit4]   2> 539384 T685 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/]
   [junit4]   2> 539385 T685 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 539385 T685 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 539385 T685 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 539386 T685 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 539386 T685 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 539386 T685 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 539386 T685 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 539387 T685 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 539387 T685 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 539387 T685 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 539388 T685 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 539388 T685 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 539388 T685 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 539388 T685 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25798/solr
   [junit4]   2> 539389 T685 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 539389 T685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 539391 T729 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7bbea410 name:ZooKeeperConnection Watcher:127.0.0.1:25798 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 539392 T685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 539395 T685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 539396 T731 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@346eb30f name:ZooKeeperConnection Watcher:127.0.0.1:25798/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 539396 T685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 539401 T685 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 540403 T685 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25805__h
   [junit4]   2> 540404 T685 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25805__h
   [junit4]   2> 540407 T718 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 540407 T732 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 540407 T711 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 540414 T733 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 540414 T733 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 540415 T711 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 540415 T733 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 540416 T708 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 540416 T708 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25805/_h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25805__h",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 540416 T708 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
   [junit4]   2> 540417 T708 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 540419 T711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 540419 T732 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 540419 T718 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 541415 T733 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 541416 T733 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 541416 T733 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 541417 T733 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 541417 T733 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 541417 T733 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/collection1/'
   [junit4]   2> 541419 T733 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 541420 T733 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 541420 T733 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 541482 T733 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 541527 T733 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 541528 T733 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 541538 T733 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 541902 T733 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 541905 T733 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 541907 T733 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 541911 T733 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 541930 T733 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 541931 T733 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 541931 T733 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/collection1/, dataDir=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty1/
   [junit4]   2> 541931 T733 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6d87b6ba
   [junit4]   2> 541932 T733 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty1
   [junit4]   2> 541932 T733 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty1/index/
   [junit4]   2> 541933 T733 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 541934 T733 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty1/index
   [junit4]   2> 541934 T733 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 541935 T733 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=BaseDirectoryWrapper(RAMDirectory@32aa968c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@541c51e3),segFN=segments_1,generation=1}
   [junit4]   2> 541935 T733 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 541941 T733 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 541941 T733 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 541942 T733 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 541942 T733 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 541942 T733 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 541942 T733 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 541943 T733 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 541943 T733 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 541943 T733 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 541944 T733 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 541945 T733 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 541945 T733 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 541946 T733 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 541946 T733 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 541947 T733 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 541947 T733 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 541947 T733 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 541948 T733 oasc.RequestHandlers.initHandlersFromConfig WARN Multiple requestHandler registered to the same name: /update ignoring: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 541948 T733 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 541949 T733 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 541949 T733 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 541967 T733 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 541969 T733 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 541971 T733 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 541973 T733 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 541975 T733 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 541979 T733 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 541979 T733 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 541980 T733 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@3f9e0f2d
   [junit4]   2> 541981 T733 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=BaseDirectoryWrapper(RAMDirectory@32aa968c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@541c51e3),segFN=segments_1,generation=1}
   [junit4]   2> 541981 T733 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 541981 T733 oass.SolrIndexSearcher.<init> Opening Searcher@3b38b6cf[collection1] main
   [junit4]   2> 541982 T733 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 541982 T733 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 541982 T733 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 541983 T733 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 541983 T733 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 541983 T733 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 541984 T733 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream Read 38 bytes from znode /configs/conf1/_rest_managed.json
   [junit4]   2> 541984 T733 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 541985 T733 oasr.ManagedResource.reloadFromStorage Loaded initArgs {} for /rest/managed
   [junit4]   2> 541985 T733 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 541989 T734 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3b38b6cf[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 541993 T733 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 541993 T737 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25805/_h collection:collection1 shard:shard1
   [junit4]   2> 541994 T685 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
   [junit4]   2> 541994 T685 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 541994 T737 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 541998 T737 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 542000 T711 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 542000 T737 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 542000 T737 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C274 name=collection1 org.apache.solr.core.SolrCore@22b0fea8 url=http://127.0.0.1:25805/_h/collection1 node=127.0.0.1:25805__h C274_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:25805/_h, core=collection1, node_name=127.0.0.1:25805__h}
   [junit4]   2> 542001 T737 C274 P25805 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25805/_h/collection1/
   [junit4]   2> 542001 T737 C274 P25805 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 542001 T708 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 542001 T737 C274 P25805 oasc.SyncStrategy.syncToMe http://127.0.0.1:25805/_h/collection1/ has no replicas
   [junit4]   2> 542001 T737 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25805/_h/collection1/ shard1
   [junit4]   2> 542002 T737 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 542003 T732 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 542003 T718 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 542003 T711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 542006 T708 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 542008 T711 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 542008 T711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 542008 T732 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 542008 T718 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 542056 T737 oasc.ZkController.register We are http://127.0.0.1:25805/_h/collection1/ and leader is http://127.0.0.1:25805/_h/collection1/
   [junit4]   2> 542057 T737 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25805/_h
   [junit4]   2> 542057 T737 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 542057 T737 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 542057 T737 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 542058 T711 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 542058 T737 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 542059 T708 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 542060 T708 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25805/_h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25805__h",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 542163 T732 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 542163 T718 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 542163 T711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 543062 T685 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 543063 T685 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 543065 T685 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25808
   [junit4]   2> 543066 T685 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 543066 T685 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 543066 T685 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004
   [junit4]   2> 543067 T685 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/'
   [junit4]   2> 543096 T685 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/solr.xml
   [junit4]   2> 543146 T685 oasc.CoreContainer.<init> New CoreContainer 117677143
   [junit4]   2> 543146 T685 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/]
   [junit4]   2> 543147 T685 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 543147 T685 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 543147 T685 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 543148 T685 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 543148 T685 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 543148 T685 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 543148 T685 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 543148 T685 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 543149 T685 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 543149 T685 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 543149 T685 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 543150 T685 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 543150 T685 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 543150 T685 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25798/solr
   [junit4]   2> 543150 T685 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 543151 T685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 543153 T748 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e0d1759 name:ZooKeeperConnection Watcher:127.0.0.1:25798 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 543154 T685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 543156 T685 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 543157 T750 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18e085c9 name:ZooKeeperConnection Watcher:127.0.0.1:25798/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 543157 T685 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 543161 T685 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 544164 T685 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25808__h
   [junit4]   2> 544165 T685 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25808__h
   [junit4]   2> 544168 T711 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 544168 T732 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 544168 T751 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 544168 T718 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 544174 T752 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 544175 T752 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 544175 T752 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 544175 T711 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 544176 T708 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 544177 T708 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25808/_h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25808__h",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 544177 T708 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 544177 T708 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 544179 T751 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 544179 T711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 544179 T718 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 544179 T732 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 545176 T752 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 545177 T752 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 545177 T752 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 545177 T752 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 545178 T752 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 545178 T752 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/collection1/'
   [junit4]   2> 545179 T752 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 545180 T752 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 545180 T752 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 545223 T752 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 545254 T752 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 545255 T752 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 545261 T752 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 545512 T752 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 545514 T752 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 545516 T752 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 545519 T752 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 545543 T752 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 545543 T752 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 545543 T752 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/collection1/, dataDir=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty2/
   [junit4]   2> 545544 T752 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6d87b6ba
   [junit4]   2> 545545 T752 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty2
   [junit4]   2> 545545 T752 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty2/index/
   [junit4]   2> 545546 T752 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 545546 T752 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty2/index
   [junit4]   2> 545547 T752 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 545548 T752 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=BaseDirectoryWrapper(RAMDirectory@3d44ce56 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@17ba34c2),segFN=segments_1,generation=1}
   [junit4]   2> 545548 T752 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 545553 T752 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 545554 T752 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 545554 T752 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 545554 T752 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 545555 T752 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 545555 T752 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 545555 T752 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 545556 T752 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 545556 T752 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 545557 T752 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 545557 T752 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 545558 T752 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 545558 T752 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 545559 T752 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 545559 T752 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 545560 T752 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 545560 T752 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 545561 T752 oasc.RequestHandlers.initHandlersFromConfig WARN Multiple requestHandler registered to the same name: /update ignoring: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 545561 T752 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 545561 T752 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 545562 T752 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 545580 T752 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 545582 T752 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 545584 T752 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 545586 T752 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 545588 T752 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 545592 T752 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 545592 T752 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 545593 T752 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@612d1e09
   [junit4]   2> 545594 T752 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=BaseDirectoryWrapper(RAMDirectory@3d44ce56 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@17ba34c2),segFN=segments_1,generation=1}
   [junit4]   2> 545594 T752 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 5

[...truncated too long message...]

elds
   [junit4]   2> 5422859 T31606 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 5422859 T31606 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1384115479
   [junit4]   2> 5422859 T31606 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@510aa80f
   [junit4]   2> 5422860 T31606 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=443,autocommits=0,soft autocommits=0,optimizes=17,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=20227,cumulative_deletesById=155,cumulative_deletesByQuery=18,cumulative_errors=0}
   [junit4]   2> 5422860 T31606 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 5422860 T31606 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 5422860 T31606 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 5422860 T31606 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 5422861 T31606 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 5422861 T31606 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.request.TestIntervalFaceting-E7864240DC5F2611-001/init-core-data-001/index [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.request.TestIntervalFaceting-E7864240DC5F2611-001/init-core-data-001/index;done=false>>]
   [junit4]   2> 5422861 T31606 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.request.TestIntervalFaceting-E7864240DC5F2611-001/init-core-data-001/index
   [junit4]   2> 5422861 T31606 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.request.TestIntervalFaceting-E7864240DC5F2611-001/init-core-data-001 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.request.TestIntervalFaceting-E7864240DC5F2611-001/init-core-data-001;done=false>>]
   [junit4]   2> 5422862 T31606 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.request.TestIntervalFaceting-E7864240DC5F2611-001/init-core-data-001
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.request.TestIntervalFaceting-E7864240DC5F2611-001
   [junit4]   2> NOTE: test params are: codec=Lucene49: {test_l_dv=FST41, test_d_dv=FST41, test_f_dv=PostingsFormat(name=FSTPulsing41), test_fs_dv=Pulsing41(freqCutoff=15 minBlockSize=44 maxBlockSize=113), test_dt_dv=FST41, test_ls_dv=PostingsFormat(name=NestedPulsing), test_ds_dv=PostingsFormat(name=NestedPulsing), test_s=FST41, id=Pulsing41(freqCutoff=15 minBlockSize=44 maxBlockSize=113)}, docValues:{test_l_dv=DocValuesFormat(name=Lucene49), test_d_dv=DocValuesFormat(name=Lucene49), test_is_dv=DocValuesFormat(name=Asserting), test_f_dv=DocValuesFormat(name=Asserting), test_fs_dv=DocValuesFormat(name=SimpleText), id_dv=DocValuesFormat(name=Asserting), test_ss_dv=DocValuesFormat(name=Lucene49), test_dt_dv=DocValuesFormat(name=Lucene49), test_ls_dv=DocValuesFormat(name=Memory), test_ds_dv=DocValuesFormat(name=Memory), test_s_dv=DocValuesFormat(name=SimpleText), test_i_dv=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=th_TH_TH_#u-nu-thai, timezone=Asia/Dacca
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=1,free=175281744,total=536346624
   [junit4]   2> NOTE: All tests run in this JVM: [TestCoreContainer, CoreContainerCoreInitFailuresTest, SolrRequestParserTest, TestBlendedInfixSuggestions, TestIndexSearcher, ResponseHeaderTest, BJQParserTest, TestLRUCache, QueryParsingTest, LukeRequestHandlerTest, CurrencyFieldXmlFileTest, TestCloudInspectUtil, TestMergePolicyConfig, TestValueSourceCache, ShardSplitTest, TestRecovery, TestNonNRTOpen, TestManagedStopFilterFactory, TestLFUCache, SliceStateTest, FullSolrCloudDistribCmdsTest, HdfsBasicDistributedZkTest, HighlighterMaxOffsetTest, SolrInfoMBeanTest, HdfsLockFactoryTest, TestPerFieldSimilarity, OverseerRolesTest, UUIDFieldTest, TestQuerySenderNoQuery, TestZkChroot, TestReversedWildcardFilterFactory, TestQueryUtils, URLClassifyProcessorTest, TestImplicitCoreProperties, AnalyticsMergeStrategyTest, SortByFunctionTest, TestSchemaNameResource, SystemInfoHandlerTest, InfoHandlerTest, TestSolrXmlPersistor, TestLeaderElectionZkExpiry, FieldAnalysisRequestHandlerTest, TestMissingGroups, DirectSolrConnectionTest, TestFiltering, DirectUpdateHandlerOptimizeTest, EnumFieldTest, BlockCacheTest, LoggingHandlerTest, TestPHPSerializedResponseWriter, TestJoin, TestDynamicFieldCollectionResource, DOMUtilTest, AlternateDirectoryTest, TestDocBasedVersionConstraints, OutputWriterTest, TestFastWriter, DebugComponentTest, TestManagedSchemaFieldResource, TestSweetSpotSimilarityFactory, SuggestComponentTest, MultiTermTest, TestExtendedDismaxParser, DocValuesTest, AutoCommitTest, TestFastLRUCache, TestBinaryField, TestCSVResponseWriter, TestLazyCores, DistributedQueryElevationComponentTest, CustomCollectionTest, ShardRoutingCustomTest, TestSystemIdResolver, TestAddFieldRealTimeGet, TestShardHandlerFactory, SpatialFilterTest, TestStressLucene, CSVRequestHandlerTest, DistributedQueryComponentCustomSortTest, TestGroupingSearch, TestDocumentBuilder, TestCopyFieldCollectionResource, RequestHandlersTest, CircularListTest, TestPhraseSuggestions, SampleTest, TestCSVLoader, TestPseudoReturnFields, PrimUtilsTest, TestSolr4Spatial2, TestUpdate, AddBlockUpdateTest, TestAtomicUpdateErrorCases, SolrIndexConfigTest, BasicZkTest, UniqFieldsUpdateProcessorFactoryTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, LeaderElectionIntegrationTest, RecoveryZkTest, LeaderElectionTest, ZkCLITest, TestFaceting, TermVectorComponentDistributedTest, TestRealTimeGet, TestMultiCoreConfBootstrap, HardAutoCommitTest, DistributedTermsComponentTest, QueryElevationComponentTest, BadIndexSchemaTest, ConvertedLegacyTest, TestFunctionQuery, DirectUpdateHandlerTest, SoftAutoCommitTest, CurrencyFieldOpenExchangeTest, SimplePostToolTest, TestCoreDiscovery, SuggesterFSTTest, CoreAdminHandlerTest, PolyFieldTest, SchemaVersionSpecificBehaviorTest, FieldMutatingUpdateProcessorTest, DefaultValueUpdateProcessorTest, DocValuesMultiTest, DistanceFunctionTest, CacheHeaderTest, TestWriterPerf, DocumentAnalysisRequestHandlerTest, TestOmitPositions, TermsComponentTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, RequiredFieldsTest, FastVectorHighlighterTest, RegexBoostProcessorTest, TestCollationField, UpdateRequestProcessorFactoryTest, TestAnalyzedSuggestions, TestPartialUpdateDeduplication, TestBinaryResponseWriter, TestSolrIndexConfig, ResponseLogComponentTest, TestStressRecovery, PreAnalyzedFieldTest, DateFieldTest, SpellingQueryConverterTest, ClusterStateTest, TestUtils, FileUtilsTest, CursorPagingTest, DistributedIntervalFacetingTest, ConnectionManagerTest, DeleteInactiveReplicaTest, DistribCursorPagingTest, SSLMigrationTest, SolrXmlInZkTest, TestDistribDocBasedVersion, TestMiniSolrCloudCluster, TestRequestStatusCollectionAPI, TestShortCircuitedRequests, HdfsChaosMonkeySafeLeaderTest, HdfsRecoveryZkTest, HdfsUnloadDistributedZkTest, CachingDirectoryFactoryTest, QueryResultKeyTest, TestConfigSets, TestInfoStreamLogging, TestReloadAndDeleteDocs, CoreAdminCreateDiscoverTest, CoreAdminRequestStatusTest, CoreMergeIndexesAdminHandlerTest, DistributedDebugComponentTest, DistributedQueryComponentOptimizationTest, DistributedSuggestComponentTest, TestIntervalFaceting]
   [junit4] Completed on J1 in 197.51s, 17 tests, 3 failures <<< FAILURES!

[...truncated 162 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:481: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:454: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/common-build.xml:496: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1308: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:923: There were test failures: 410 suites, 1711 tests, 1 error, 3 failures, 50 ignored (8 assumptions)

Total time: 281 minutes 47 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-4.x #586
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 462708180 bytes
Compression is 0.0%
Took 1 min 30 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 597 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/597/

1 tests failed.
REGRESSION:  org.apache.lucene.util.automaton.TestAutomaton.testRandomFinite

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([300BCC40FDA94734:77BDAA7D42392215]:0)
	at java.lang.Integer.valueOf(Integer.java:642)
	at org.apache.lucene.util.automaton.MinimizationOperations.minimizeHopcroft(MinimizationOperations.java:110)
	at org.apache.lucene.util.automaton.MinimizationOperations.minimize(MinimizationOperations.java:51)
	at org.apache.lucene.util.automaton.TestAutomaton.randomNoOp(TestAutomaton.java:586)
	at org.apache.lucene.util.automaton.TestAutomaton.unionTerms(TestAutomaton.java:639)
	at org.apache.lucene.util.automaton.TestAutomaton.assertSame(TestAutomaton.java:1050)
	at org.apache.lucene.util.automaton.TestAutomaton.testRandomFinite(TestAutomaton.java:1003)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)




Build Log:
[...truncated 743 lines...]
   [junit4] Suite: org.apache.lucene.util.automaton.TestAutomaton
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestAutomaton -Dtests.method=testRandomFinite -Dtests.seed=300BCC40FDA94734 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/hudson/lucene-data/enwiki.random.lines.txt -Dtests.locale=tr_TR -Dtests.timezone=America/Maceio -Dtests.file.encoding=UTF-8
   [junit4] ERROR   55.9s J1 | TestAutomaton.testRandomFinite <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([300BCC40FDA94734:77BDAA7D42392215]:0)
   [junit4]    > 	at java.lang.Integer.valueOf(Integer.java:642)
   [junit4]    > 	at org.apache.lucene.util.automaton.MinimizationOperations.minimizeHopcroft(MinimizationOperations.java:110)
   [junit4]    > 	at org.apache.lucene.util.automaton.MinimizationOperations.minimize(MinimizationOperations.java:51)
   [junit4]    > 	at org.apache.lucene.util.automaton.TestAutomaton.randomNoOp(TestAutomaton.java:586)
   [junit4]    > 	at org.apache.lucene.util.automaton.TestAutomaton.unionTerms(TestAutomaton.java:639)
   [junit4]    > 	at org.apache.lucene.util.automaton.TestAutomaton.assertSame(TestAutomaton.java:1050)
   [junit4]    > 	at org.apache.lucene.util.automaton.TestAutomaton.testRandomFinite(TestAutomaton.java:1003)
   [junit4]   2> NOTE: test params are: codec=Lucene3x, sim=DefaultSimilarity, locale=tr_TR, timezone=America/Maceio
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=1,free=437182832,total=460324864
   [junit4]   2> NOTE: All tests run in this JVM: [TestSnapshotDeletionPolicy, TestPerFieldPostingsFormat, TestMultiMMap, TestSameScoresWithThreads, TestMixedDocValuesUpdates, TestCodecHoldsOpenFiles, TestFieldReuse, TestMutableValues, TestLucene41StoredFieldsFormat, TestExternalCodecs, TestPositiveScoresOnlyCollector, Test2BNumericDocValues, TestBufferedIndexInput, TestDocValuesIndexing, TestNIOFSDirectory, TestOrdinalMap, TestLucene40TermVectorsFormat, TestPayloads, TestIndexWriterOnDiskFull, TestTryDelete, TestBinaryDocument, TestStoredFieldsFormat, TestBackwardsCompatibility3x, TestParallelAtomicReader, TestIndexReaderClose, TestFuzzyQuery, TestNumericTokenStream, Before3, Before3, TestIntsRef, TestWeakIdentityMap, TestFieldCache, TestCompressingStoredFieldsFormat, TestTermRangeFilter, TestSimpleExplanationsOfNonMatches, TestNGramPhraseQuery, TestDoubleBarrelLRUCache, TestTotalHitCountCollector, TestByteBlockPool, TestField, TestTerm, TestAutomaton]
   [junit4] Completed on J1 in 56.90s, 42 tests, 1 error <<< FAILURES!

[...truncated 1347 lines...]
   [junit4] JVM J1: stdout was not empty, see: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/build/core/test/temp/junit4-J1-20140814_075352_065.sysout
   [junit4] >>> JVM J1: stdout (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/heapdumps/java_pid6784.hprof ...
   [junit4] Heap dump file created [790904353 bytes in 10.449 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 15 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:481: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:454: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/build.xml:49: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1308: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:923: There were test failures: 418 suites, 3578 tests, 1 error, 153 ignored (142 assumptions)

Total time: 82 minutes 49 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-4.x #586
Archived 4 artifacts
Archive block size is 32768
Received 0 blocks and 795967258 bytes
Compression is 0.0%
Took 2 min 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 596 - Still Failing

Posted by Tomás Fernández Löbbe <to...@gmail.com>.
I can reproduce the failure of TestintervalFaceting on branch 4x. I'm
looking at this now.

Tomás


On Tue, Aug 12, 2014 at 8:55 PM, Apache Jenkins Server <
jenkins@builds.apache.org> wrote:

> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/596/
>
> 4 tests failed.
> FAILED:
>  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testDistribSearch
>
> Error Message:
> IOException occured when talking to server at:
> http://127.0.0.1:25801/_h/collection1
>
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException: IOException occured when
> talking to server at: http://127.0.0.1:25801/_h/collection1
>         at
> __randomizedtesting.SeedInfo.seed([E7864240DC5F2611:6660CC58AB00462D]:0)
>         at
> org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:562)
>         at
> org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
>         at
> org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
>         at
> org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
>         at
> org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
>         at
> org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:223)
>         at
> org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:165)
>         at
> org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrServer(FullSolrCloudDistribCmdsTest.java:414)
>         at
> org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.doTest(FullSolrCloudDistribCmdsTest.java:144)
>         at
> org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:867)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>         at
> org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at
> org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>         at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at
> org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>         at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at
> org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>         at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at
> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: java.net.SocketException: Broken pipe
>         at java.net.SocketOutputStream.socketWrite0(Native Method)
>         at
> java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113)
>         at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
>         at
> org.apache.http.impl.io.AbstractSessionOutputBuffer.flushBuffer(AbstractSessionOutputBuffer.java:159)
>         at
> org.apache.http.impl.io.AbstractSessionOutputBuffer.writeLine(AbstractSessionOutputBuffer.java:258)
>         at
> org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:59)
>         at
> org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:46)
>         at
> org.apache.http.impl.io.AbstractMessageWriter.write(AbstractMessageWriter.java:109)
>         at
> org.apache.http.impl.AbstractHttpClientConnection.sendRequestHeader(AbstractHttpClientConnection.java:254)
>         at
> org.apache.http.impl.conn.DefaultClientConnection.sendRequestHeader(DefaultClientConnection.java:270)
>         at
> org.apache.http.impl.conn.ManagedClientConnectionImpl.sendRequestHeader(ManagedClientConnectionImpl.java:209)
>         at
> org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:203)
>         at
> org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:121)
>         at
> org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
>         at
> org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
>         at
> org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
>         at
> org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
>         at
> org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
>         at
> org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
>         at
> org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:448)
>         ... 50 more
>
>
> FAILED:
>  org.apache.solr.request.TestIntervalFaceting.testIntFieldsMultipleSegments
>
> Error Message:
> Expected multiple reader leaves. Found 1
>
> Stack Trace:
> java.lang.AssertionError: Expected multiple reader leaves. Found 1
>         at
> __randomizedtesting.SeedInfo.seed([E7864240DC5F2611:E225149F98057EC8]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.junit.Assert.assertTrue(Assert.java:43)
>         at
> org.apache.solr.request.TestIntervalFaceting.assertMultipleReaders(TestIntervalFaceting.java:134)
>         at
> org.apache.solr.request.TestIntervalFaceting.testIntFieldsMultipleSegments(TestIntervalFaceting.java:711)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>         at
> org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at
> org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>         at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at
> org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>         at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at
> org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>         at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at
> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> FAILED:  org.apache.solr.request.TestIntervalFaceting.testMultipleSegments
>
> Error Message:
> Expected multiple reader leaves. Found 1
>
> Stack Trace:
> java.lang.AssertionError: Expected multiple reader leaves. Found 1
>         at
> __randomizedtesting.SeedInfo.seed([E7864240DC5F2611:5BF1F7191BF88A54]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.junit.Assert.assertTrue(Assert.java:43)
>         at
> org.apache.solr.request.TestIntervalFaceting.assertMultipleReaders(TestIntervalFaceting.java:134)
>         at
> org.apache.solr.request.TestIntervalFaceting.testMultipleSegments(TestIntervalFaceting.java:103)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>         at
> org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at
> org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>         at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at
> org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>         at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at
> org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>         at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at
> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> FAILED:  org.apache.solr.request.TestIntervalFaceting.testMultiValueFields
>
> Error Message:
> Expected multiple reader leaves. Found 1
>
> Stack Trace:
> java.lang.AssertionError: Expected multiple reader leaves. Found 1
>         at
> __randomizedtesting.SeedInfo.seed([E7864240DC5F2611:EE26DEDEBE485AEB]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.junit.Assert.assertTrue(Assert.java:43)
>         at
> org.apache.solr.request.TestIntervalFaceting.assertMultipleReaders(TestIntervalFaceting.java:134)
>         at
> org.apache.solr.request.TestIntervalFaceting.testMultiValueFields(TestIntervalFaceting.java:71)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>         at
> org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at
> org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>         at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at
> org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>         at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at
> org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>         at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at
> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
>
>
> Build Log:
> [...truncated 11757 lines...]
>    [junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
>    [junit4]   2> Creating dataDir:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/init-core-data-001
>    [junit4]   2> 535000 T685
> oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext
> system property: /_h/
>    [junit4]   2> 535006 T685 oas.SolrTestCaseJ4.setUp ###Starting
> testDistribSearch
>    [junit4]   2> 535007 T685 oasc.ZkTestServer.run STARTING ZK TEST SERVER
>    [junit4]   1> client port:0.0.0.0/0.0.0.0:0
>    [junit4]   2> 535008 T686 oasc.ZkTestServer$ZKServerMain.runFromConfig
> Starting server
>    [junit4]   2> 535108 T685 oasc.ZkTestServer.run start zk server on
> port:25798
>    [junit4]   2> 535109 T685 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 535115 T692 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@3bc4520
> name:ZooKeeperConnection Watcher:127.0.0.1:25798 got event WatchedEvent
> state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 535115 T685 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 535115 T685 oascc.SolrZkClient.makePath makePath: /solr
>    [junit4]   2> 535120 T685 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 535121 T694 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@189513ae
> name:ZooKeeperConnection Watcher:127.0.0.1:25798/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 535121 T685 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 535121 T685 oascc.SolrZkClient.makePath makePath:
> /collections/collection1
>    [junit4]   2> 535124 T685 oascc.SolrZkClient.makePath makePath:
> /collections/collection1/shards
>    [junit4]   2> 535125 T685 oascc.SolrZkClient.makePath makePath:
> /collections/control_collection
>    [junit4]   2> 535126 T685 oascc.SolrZkClient.makePath makePath:
> /collections/control_collection/shards
>    [junit4]   2> 535128 T685 oasc.AbstractZkTestCase.putConfig put
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
> to /configs/conf1/solrconfig.xml
>    [junit4]   2> 535129 T685 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/solrconfig.xml
>    [junit4]   2> 535131 T685 oasc.AbstractZkTestCase.putConfig put
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/schema15.xml
> to /configs/conf1/schema.xml
>    [junit4]   2> 535132 T685 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/schema.xml
>    [junit4]   2> 535134 T685 oasc.AbstractZkTestCase.putConfig put
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
> to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>    [junit4]   2> 535134 T685 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>    [junit4]   2> 535136 T685 oasc.AbstractZkTestCase.putConfig put
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
> to /configs/conf1/stopwords.txt
>    [junit4]   2> 535137 T685 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/stopwords.txt
>    [junit4]   2> 535138 T685 oasc.AbstractZkTestCase.putConfig put
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt
> to /configs/conf1/protwords.txt
>    [junit4]   2> 535139 T685 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/protwords.txt
>    [junit4]   2> 535141 T685 oasc.AbstractZkTestCase.putConfig put
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/currency.xml
> to /configs/conf1/currency.xml
>    [junit4]   2> 535141 T685 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/currency.xml
>    [junit4]   2> 535143 T685 oasc.AbstractZkTestCase.putConfig put
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
> to /configs/conf1/enumsConfig.xml
>    [junit4]   2> 535144 T685 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/enumsConfig.xml
>    [junit4]   2> 535145 T685 oasc.AbstractZkTestCase.putConfig put
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
> to /configs/conf1/open-exchange-rates.json
>    [junit4]   2> 535146 T685 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/open-exchange-rates.json
>    [junit4]   2> 535148 T685 oasc.AbstractZkTestCase.putConfig put
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
> to /configs/conf1/mapping-ISOLatin1Accent.txt
>    [junit4]   2> 535148 T685 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/mapping-ISOLatin1Accent.txt
>    [junit4]   2> 535150 T685 oasc.AbstractZkTestCase.putConfig put
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
> to /configs/conf1/old_synonyms.txt
>    [junit4]   2> 535151 T685 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/old_synonyms.txt
>    [junit4]   2> 535152 T685 oasc.AbstractZkTestCase.putConfig put
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
> to /configs/conf1/synonyms.txt
>    [junit4]   2> 535153 T685 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/synonyms.txt
>    [junit4]   2> 536344 T685 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 536349 T685 oejs.AbstractConnector.doStart Started
> SocketConnector@127.0.0.1:25801
>    [junit4]   2> 536349 T685 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()
>    [junit4]   2> 536350 T685 oasc.SolrResourceLoader.locateSolrHome JNDI
> not configured for solr (NoInitialContextEx)
>    [junit4]   2> 536350 T685 oasc.SolrResourceLoader.locateSolrHome using
> system property solr.solr.home:
> ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002
>    [junit4]   2> 536350 T685 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/'
>    [junit4]   2> 536380 T685 oasc.ConfigSolr.fromFile Loading container
> configuration from
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/solr.xml
>    [junit4]   2> 536430 T685 oasc.CoreContainer.<init> New CoreContainer
> 64529212
>    [junit4]   2> 536430 T685 oasc.CoreContainer.load Loading cores into
> CoreContainer
> [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/]
>    [junit4]   2> 536431 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 90000
>    [junit4]   2> 536431 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to:
>    [junit4]   2> 536431 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>    [junit4]   2> 536432 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>    [junit4]   2> 536432 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>    [junit4]   2> 536432 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>    [junit4]   2> 536432 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>    [junit4]   2> 536433 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>    [junit4]   2> 536433 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>    [junit4]   2> 536433 T685 oasu.UpdateShardHandler.<init> Creating
> UpdateShardHandler HTTP client with params:
> socketTimeout=340000&connTimeout=45000&retry=false
>    [junit4]   2> 536434 T685 oasl.LogWatcher.createWatcher SLF4J impl is
> org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 536434 T685 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 536434 T685 oasc.CoreContainer.load Host Name: 127.0.0.1
>    [junit4]   2> 536434 T685 oasc.ZkContainer.initZooKeeper Zookeeper
> client=127.0.0.1:25798/solr
>    [junit4]   2> 536435 T685 oasc.ZkController.checkChrootPath zkHost
> includes chroot
>    [junit4]   2> 536435 T685 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 536437 T705 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@3c4ba1d
> name:ZooKeeperConnection Watcher:127.0.0.1:25798 got event WatchedEvent
> state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 536438 T685 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 536441 T685 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 536442 T707 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@93cb9b4
> name:ZooKeeperConnection Watcher:127.0.0.1:25798/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 536442 T685 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 536444 T685 oascc.SolrZkClient.makePath makePath:
> /overseer/queue
>    [junit4]   2> 536446 T685 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-queue-work
>    [junit4]   2> 536448 T685 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-map-running
>    [junit4]   2> 536449 T685 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-map-completed
>    [junit4]   2> 536451 T685 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-map-failure
>    [junit4]   2> 536453 T685 oascc.SolrZkClient.makePath makePath:
> /live_nodes
>    [junit4]   2> 536454 T685 oasc.ZkController.createEphemeralLiveNode
> Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25801__h
>    [junit4]   2> 536455 T685 oascc.SolrZkClient.makePath makePath:
> /live_nodes/127.0.0.1:25801__h
>    [junit4]   2> 536457 T685 oascc.SolrZkClient.makePath makePath:
> /overseer_elect
>    [junit4]   2> 536458 T685 oascc.SolrZkClient.makePath makePath:
> /overseer_elect/election
>    [junit4]   2> 536460 T685 oasc.OverseerElectionContext.runLeaderProcess
> I am going to be the leader 127.0.0.1:25801__h
>    [junit4]   2> 536460 T685 oascc.SolrZkClient.makePath makePath:
> /overseer_elect/leader
>    [junit4]   2> 536462 T685 oasc.Overseer.start Overseer
> (id=92267852054724611-127.0.0.1:25801__h-n_0000000000) starting
>    [junit4]   2> 536463 T685 oascc.SolrZkClient.makePath makePath:
> /overseer/queue-work
>    [junit4]   2> 536469 T709 oasc.OverseerCollectionProcessor.run Process
> current queue of collection creations
>    [junit4]   2> 536469 T685 oascc.SolrZkClient.makePath makePath:
> /clusterstate.json
>    [junit4]   2> 536471 T685 oascc.SolrZkClient.makePath makePath:
> /aliases.json
>    [junit4]   2> 536472 T685
> oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
> state from ZooKeeper...
>    [junit4]   2> 536473 T708 oasc.Overseer$ClusterStateUpdater.run
> Starting to work on the main queue
>    [junit4]   2> 536478 T710 oasc.ZkController.publish publishing
> core=collection1 state=down collection=control_collection
>    [junit4]   2> 536478 T710 oasc.ZkController.publish numShards not found
> on descriptor - reading it from system property
>    [junit4]   2> 536479 T710 oasc.ZkController.waitForCoreNodeName look
> for our core node name
>    [junit4]   2> 536479 T711
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 536480 T708 oascc.ZkStateReader.updateClusterState
> Updating cloud state from ZooKeeper...
>    [junit4]   2> 536481 T708 oasc.Overseer$ClusterStateUpdater.updateState
> Update state numShards=1 message={
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "base_url":"http://127.0.0.1:25801/_h",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:25801__h",
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "numShards":"1",
>    [junit4]   2>          "core_node_name":null}
>    [junit4]   2> 536481 T708
> oasc.Overseer$ClusterStateUpdater.createCollection Create collection
> control_collection with shards [shard1]
>    [junit4]   2> 536481 T708 oasc.Overseer$ClusterStateUpdater.updateState
> Assigning new node to shard shard=shard1
>    [junit4]   2> 536483 T711 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 537479 T710 oasc.ZkController.waitForShardId waiting to
> find shard id in clusterstate for collection1
>    [junit4]   2> 537480 T710 oasc.ZkController.createCollectionZkNode
> Check for collection zkNode:control_collection
>    [junit4]   2> 537480 T710 oasc.ZkController.createCollectionZkNode
> Collection zkNode exists
>    [junit4]   2> 537480 T710 oascc.ZkStateReader.readConfigName Load
> collection config from:/collections/control_collection
>    [junit4]   2> 537481 T710 oascc.ZkStateReader.readConfigName
> path=/collections/control_collection configName=conf1 specified config
> exists in ZooKeeper
>    [junit4]   2> 537481 T710 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/collection1/'
>    [junit4]   2> 537483 T710 oasc.SolrResourceLoader.replaceClassLoader
> Adding
> 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/collection1/lib/README'
> to classloader
>    [junit4]   2> 537484 T710 oasc.SolrResourceLoader.replaceClassLoader
> Adding
> 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/collection1/lib/classes/'
> to classloader
>    [junit4]   2> 537485 T710 oasc.SolrResourceLoader.replaceClassLoader
> Adding
> 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/collection1/lib/.svn/'
> to classloader
>    [junit4]   2> 537548 T710 oasc.SolrConfig.<init> Using Lucene
> MatchVersion: LUCENE_4_10
>    [junit4]   2> 537593 T710 oasc.SolrConfig.<init> Loaded SolrConfig:
> solrconfig.xml
>    [junit4]   2> 537595 T710 oass.IndexSchema.readSchema Reading Solr
> Schema from schema.xml
>    [junit4]   2> 537604 T710 oass.IndexSchema.readSchema [collection1]
> Schema name=test
>    [junit4]   2> 537976 T710 oass.IndexSchema.readSchema default search
> field in schema is text
>    [junit4]   2> 537979 T710 oass.IndexSchema.readSchema unique key field:
> id
>    [junit4]   2> 537981 T710 oass.FileExchangeRateProvider.reload
> Reloading exchange rates from file currency.xml
>    [junit4]   2> 537985 T710 oass.FileExchangeRateProvider.reload
> Reloading exchange rates from file currency.xml
>    [junit4]   2> 538004 T710 oasc.CoreContainer.create Creating SolrCore
> 'collection1' using configuration from collection control_collection
>    [junit4]   2> 538005 T710 oasc.SolrCore.initDirectoryFactory
> org.apache.solr.core.MockDirectoryFactory
>    [junit4]   2> 538005 T710 oasc.SolrCore.<init> [collection1] Opening
> new SolrCore at
> ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-002/collection1/,
> dataDir=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/control/data/
>    [junit4]   2> 538005 T710 oasc.JmxMonitoredMap.<init> JMX monitoring is
> enabled. Adding Solr mbeans to JMX Server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@6d87b6ba
>    [junit4]   2> 538006 T710 oasc.CachingDirectoryFactory.get return new
> directory for
> ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/control/data
>    [junit4]   2> 538006 T710 oasc.SolrCore.getNewIndexDir New index
> directory detected: old=null
> new=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/control/data/index/
>    [junit4]   2> 538007 T710 oasc.SolrCore.initIndex WARN [collection1]
> Solr index directory
> './solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/control/data/index'
> doesn't exist. Creating new index...
>    [junit4]   2> 538008 T710 oasc.CachingDirectoryFactory.get return new
> directory for
> ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/control/data/index
>    [junit4]   2> 538008 T710 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy:
> [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22,
> maxMergeSize=9223372036854775807,
> maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=true, maxMergeDocs=2147483647,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
>    [junit4]   2> 538009 T710 oasc.SolrDeletionPolicy.onCommit
> SolrDeletionPolicy.onCommit: commits: num=1
>    [junit4]   2>
>  commit{dir=BaseDirectoryWrapper(RAMDirectory@33b526df
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1b26d6ff
> ),segFN=segments_1,generation=1}
>    [junit4]   2> 538009 T710 oasc.SolrDeletionPolicy.updateCommits newest
> commit generation = 1
>    [junit4]   2> 538014 T710 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "nodistrib"
>    [junit4]   2> 538014 T710 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "dedupe"
>    [junit4]   2> 538014 T710 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "dedupe"
>    [junit4]   2> 538015 T710 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 538015 T710 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 538015 T710 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>    [junit4]   2> 538016 T710 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 538016 T710 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 538016 T710 oasc.SolrCore.loadUpdateProcessorChains no
> updateRequestProcessorChain defined as default, creating implicit default
>    [junit4]   2> 538017 T710 oasc.RequestHandlers.initHandlersFromConfig
> created /update: org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 538017 T710 oasc.RequestHandlers.initHandlersFromConfig
> created /update/json: org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 538018 T710 oasc.RequestHandlers.initHandlersFromConfig
> created /update/csv: org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 538018 T710 oasc.RequestHandlers.initHandlersFromConfig
> created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 538019 T710 oasc.RequestHandlers.initHandlersFromConfig
> created standard: solr.StandardRequestHandler
>    [junit4]   2> 538019 T710 oasc.RequestHandlers.initHandlersFromConfig
> created /get: solr.RealTimeGetHandler
>    [junit4]   2> 538020 T710 oasc.RequestHandlers.initHandlersFromConfig
> adding lazy requestHandler: solr.ReplicationHandler
>    [junit4]   2> 538020 T710 oasc.RequestHandlers.initHandlersFromConfig
> created /replication: solr.ReplicationHandler
>    [junit4]   2> 538020 T710 oasc.RequestHandlers.initHandlersFromConfig
> WARN Multiple requestHandler registered to the same name: /update ignoring:
> org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 538021 T710 oasc.RequestHandlers.initHandlersFromConfig
> created /update: solr.UpdateRequestHandler
>    [junit4]   2> 538021 T710 oasc.RequestHandlers.initHandlersFromConfig
> created /admin/: org.apache.solr.handler.admin.AdminHandlers
>    [junit4]   2> 538022 T710 oasc.RequestHandlers.initHandlersFromConfig
> created /admin/file: solr.admin.ShowFileRequestHandler
>    [junit4]   2> 538040 T710 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 538042 T710 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 538044 T710 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 538046 T710 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 538048 T710 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 538053 T710 oasu.CommitTracker.<init> Hard AutoCommit:
> disabled
>    [junit4]   2> 538054 T710 oasu.CommitTracker.<init> Soft AutoCommit:
> disabled
>    [junit4]   2> 538054 T710 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.MockRandomMergePolicy:
> org.apache.lucene.index.MockRandomMergePolicy@1760655f
>    [junit4]   2> 538055 T710 oasc.SolrDeletionPolicy.onInit
> SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>
>  commit{dir=BaseDirectoryWrapper(RAMDirectory@33b526df
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1b26d6ff
> ),segFN=segments_1,generation=1}
>    [junit4]   2> 538055 T710 oasc.SolrDeletionPolicy.updateCommits newest
> commit generation = 1
>    [junit4]   2> 538055 T710 oass.SolrIndexSearcher.<init> Opening
> Searcher@9342108[collection1] main
>    [junit4]   2> 538056 T710 oascc.ZkStateReader.readConfigName Load
> collection config from:/collections/control_collection
>    [junit4]   2> 538056 T710 oascc.ZkStateReader.readConfigName
> path=/collections/control_collection configName=conf1 specified config
> exists in ZooKeeper
>    [junit4]   2> 538057 T710 oasr.ManagedResourceStorage.newStorageIO
> Setting up ZooKeeper-based storage for the RestManager with znodeBase:
> /configs/conf1
>    [junit4]   2> 538057 T710
> oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured
> ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 538057 T710 oasr.RestManager.init Initializing
> RestManager with initArgs: {}
>    [junit4]   2> 538058 T710 oasr.ManagedResourceStorage.load Reading
> _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 538058 T710
> oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data
> found for znode /configs/conf1/_rest_managed.json
>    [junit4]   2> 538058 T710 oasr.ManagedResourceStorage.load Loaded null
> at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 538059 T710 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/_rest_managed.json
>    [junit4]   2> 538061 T710
> oasr.ManagedResourceStorage$ZooKeeperStorageIO$1.close Wrote 38 bytes to
> new znode /configs/conf1/_rest_managed.json
>    [junit4]   2> 538061 T710 oasr.ManagedResourceStorage$JsonStorage.store
> Saved JSON object to path _rest_managed.json using
> ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 538061 T710 oasr.RestManager.init Initializing 0
> registered ManagedResources
>    [junit4]   2> 538064 T712 oasc.SolrCore.registerSearcher [collection1]
> Registered new searcher Searcher@9342108[collection1]
> main{StandardDirectoryReader(segments_1:1:nrt)}
>    [junit4]   2> 538068 T710 oasc.CoreContainer.registerCore registering
> core: collection1
>    [junit4]   2> 538068 T715 oasc.ZkController.register Register replica -
> core:collection1 address:http://127.0.0.1:25801/_h
> collection:control_collection shard:shard1
>    [junit4]   2> 538069 T685 oass.SolrDispatchFilter.init
> user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
>    [junit4]   2> 538069 T685 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init() done
>    [junit4]   2> 538069 T715 oascc.SolrZkClient.makePath makePath:
> /collections/control_collection/leader_elect/shard1/election
>    [junit4]   2> 538070 T685 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 538072 T717 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@5ab4a786
> name:ZooKeeperConnection Watcher:127.0.0.1:25798/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 538072 T685 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 538073 T685
> oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
> state from ZooKeeper...
>    [junit4]   2> 538075 T715
> oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process
> for shard shard1
>    [junit4]   2> 538075 T685 oasc.ChaosMonkey.monkeyLog monkey: init -
> expire sessions:false cause connection loss:false
>    [junit4]   2> 538076 T711
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 538077 T715
> oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas
> found to continue.
>    [junit4]   2> 538077 T715
> oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader -
> try and sync
>    [junit4]   2> ASYNC  NEW_CORE C273 name=collection1
> org.apache.solr.core.SolrCore@5fb62a19 url=
> http://127.0.0.1:25801/_h/collection1 node=127.0.0.1:25801__h
> C273_STATE=coll:control_collection core:collection1 props:{state=down,
> base_url=http://127.0.0.1:25801/_h, core=collection1,
> node_name=127.0.0.1:25801__h}
>    [junit4]   2> 538077 T715 C273 P25801 oasc.SyncStrategy.sync Sync
> replicas to http://127.0.0.1:25801/_h/collection1/
>    [junit4]   2> 538077 T708 oascc.ZkStateReader.updateClusterState
> Updating cloud state from ZooKeeper...
>    [junit4]   2> 538077 T715 C273 P25801 oasc.SyncStrategy.syncReplicas
> Sync Success - now sync replicas to me
>    [junit4]   2> 538078 T715 C273 P25801 oasc.SyncStrategy.syncToMe
> http://127.0.0.1:25801/_h/collection1/ has no replicas
>    [junit4]   2> 538078 T715
> oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader:
> http://127.0.0.1:25801/_h/collection1/ shard1
>    [junit4]   2> 538078 T715 oascc.SolrZkClient.makePath makePath:
> /collections/control_collection/leaders/shard1
>    [junit4]   2> 538080 T711 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 538080 T718 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 538083 T708 oascc.ZkStateReader.updateClusterState
> Updating cloud state from ZooKeeper...
>    [junit4]   2> 538085 T711
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 538086 T718 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 538086 T711 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 538133 T715 oasc.ZkController.register We are
> http://127.0.0.1:25801/_h/collection1/ and leader is
> http://127.0.0.1:25801/_h/collection1/
>    [junit4]   2> 538134 T715 oasc.ZkController.register No LogReplay
> needed for core=collection1 baseURL=http://127.0.0.1:25801/_h
>    [junit4]   2> 538134 T715 oasc.ZkController.checkRecovery I am the
> leader, no recovery necessary
>    [junit4]   2> 538134 T715 oasc.ZkController.publish publishing
> core=collection1 state=active collection=control_collection
>    [junit4]   2> 538134 T715 oasc.ZkController.publish numShards not found
> on descriptor - reading it from system property
>    [junit4]   2> 538136 T711
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 538136 T715 oascc.ZkStateReader.updateClusterState
> Updating cloud state from ZooKeeper...
>    [junit4]   2> 538137 T708 oascc.ZkStateReader.updateClusterState
> Updating cloud state from ZooKeeper...
>    [junit4]   2> 538137 T708 oasc.Overseer$ClusterStateUpdater.updateState
> Update state numShards=3 message={
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "state":"active",
>    [junit4]   2>          "base_url":"http://127.0.0.1:25801/_h",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:25801__h",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "numShards":"3",
>    [junit4]   2>          "core_node_name":"core_node1"}
>    [junit4]   2> 538240 T711 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 538240 T718 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 539289 T685
> oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
>    [junit4]   2> 539290 T685 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 539293 T685 oejs.AbstractConnector.doStart Started
> SocketConnector@127.0.0.1:25805
>    [junit4]   2> 539294 T685 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()
>    [junit4]   2> 539294 T685 oasc.SolrResourceLoader.locateSolrHome JNDI
> not configured for solr (NoInitialContextEx)
>    [junit4]   2> 539294 T685 oasc.SolrResourceLoader.locateSolrHome using
> system property solr.solr.home:
> ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003
>    [junit4]   2> 539294 T685 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/'
>    [junit4]   2> 539323 T685 oasc.ConfigSolr.fromFile Loading container
> configuration from
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/solr.xml
>    [junit4]   2> 539384 T685 oasc.CoreContainer.<init> New CoreContainer
> 1044030314
>    [junit4]   2> 539384 T685 oasc.CoreContainer.load Loading cores into
> CoreContainer
> [instanceDir=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/]
>    [junit4]   2> 539385 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 90000
>    [junit4]   2> 539385 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to:
>    [junit4]   2> 539385 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>    [junit4]   2> 539386 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>    [junit4]   2> 539386 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>    [junit4]   2> 539386 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>    [junit4]   2> 539386 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>    [junit4]   2> 539387 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>    [junit4]   2> 539387 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>    [junit4]   2> 539387 T685 oasu.UpdateShardHandler.<init> Creating
> UpdateShardHandler HTTP client with params:
> socketTimeout=340000&connTimeout=45000&retry=false
>    [junit4]   2> 539388 T685 oasl.LogWatcher.createWatcher SLF4J impl is
> org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 539388 T685 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 539388 T685 oasc.CoreContainer.load Host Name: 127.0.0.1
>    [junit4]   2> 539388 T685 oasc.ZkContainer.initZooKeeper Zookeeper
> client=127.0.0.1:25798/solr
>    [junit4]   2> 539389 T685 oasc.ZkController.checkChrootPath zkHost
> includes chroot
>    [junit4]   2> 539389 T685 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 539391 T729 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@7bbea410
> name:ZooKeeperConnection Watcher:127.0.0.1:25798 got event WatchedEvent
> state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 539392 T685 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 539395 T685 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 539396 T731 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@346eb30f
> name:ZooKeeperConnection Watcher:127.0.0.1:25798/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 539396 T685 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 539401 T685
> oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
> state from ZooKeeper...
>    [junit4]   2> 540403 T685 oasc.ZkController.createEphemeralLiveNode
> Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25805__h
>    [junit4]   2> 540404 T685 oascc.SolrZkClient.makePath makePath:
> /live_nodes/127.0.0.1:25805__h
>    [junit4]   2> 540407 T718 oascc.ZkStateReader$3.process Updating live
> nodes... (2)
>    [junit4]   2> 540407 T732 oascc.ZkStateReader$3.process Updating live
> nodes... (2)
>    [junit4]   2> 540407 T711 oascc.ZkStateReader$3.process Updating live
> nodes... (2)
>    [junit4]   2> 540414 T733 oasc.ZkController.publish publishing
> core=collection1 state=down collection=collection1
>    [junit4]   2> 540414 T733 oasc.ZkController.publish numShards not found
> on descriptor - reading it from system property
>    [junit4]   2> 540415 T711
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 540415 T733 oasc.ZkController.waitForCoreNodeName look
> for our core node name
>    [junit4]   2> 540416 T708 oascc.ZkStateReader.updateClusterState
> Updating cloud state from ZooKeeper...
>    [junit4]   2> 540416 T708 oasc.Overseer$ClusterStateUpdater.updateState
> Update state numShards=3 message={
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "base_url":"http://127.0.0.1:25805/_h",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:25805__h",
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "collection":"collection1",
>    [junit4]   2>          "numShards":"3",
>    [junit4]   2>          "core_node_name":null}
>    [junit4]   2> 540416 T708
> oasc.Overseer$ClusterStateUpdater.createCollection Create collection
> collection1 with shards [shard1, shard2, shard3]
>    [junit4]   2> 540417 T708 oasc.Overseer$ClusterStateUpdater.updateState
> Assigning new node to shard shard=shard1
>    [junit4]   2> 540419 T711 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 540419 T732 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 540419 T718 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 541415 T733 oasc.ZkController.waitForShardId waiting to
> find shard id in clusterstate for collection1
>    [junit4]   2> 541416 T733 oasc.ZkController.createCollectionZkNode
> Check for collection zkNode:collection1
>    [junit4]   2> 541416 T733 oasc.ZkController.createCollectionZkNode
> Collection zkNode exists
>    [junit4]   2> 541417 T733 oascc.ZkStateReader.readConfigName Load
> collection config from:/collections/collection1
>    [junit4]   2> 541417 T733 oascc.ZkStateReader.readConfigName
> path=/collections/collection1 configName=conf1 specified config exists in
> ZooKeeper
>    [junit4]   2> 541417 T733 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/collection1/'
>    [junit4]   2> 541419 T733 oasc.SolrResourceLoader.replaceClassLoader
> Adding
> 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/collection1/lib/README'
> to classloader
>    [junit4]   2> 541420 T733 oasc.SolrResourceLoader.replaceClassLoader
> Adding
> 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/collection1/lib/.svn/'
> to classloader
>    [junit4]   2> 541420 T733 oasc.SolrResourceLoader.replaceClassLoader
> Adding
> 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/collection1/lib/classes/'
> to classloader
>    [junit4]   2> 541482 T733 oasc.SolrConfig.<init> Using Lucene
> MatchVersion: LUCENE_4_10
>    [junit4]   2> 541527 T733 oasc.SolrConfig.<init> Loaded SolrConfig:
> solrconfig.xml
>    [junit4]   2> 541528 T733 oass.IndexSchema.readSchema Reading Solr
> Schema from schema.xml
>    [junit4]   2> 541538 T733 oass.IndexSchema.readSchema [collection1]
> Schema name=test
>    [junit4]   2> 541902 T733 oass.IndexSchema.readSchema default search
> field in schema is text
>    [junit4]   2> 541905 T733 oass.IndexSchema.readSchema unique key field:
> id
>    [junit4]   2> 541907 T733 oass.FileExchangeRateProvider.reload
> Reloading exchange rates from file currency.xml
>    [junit4]   2> 541911 T733 oass.FileExchangeRateProvider.reload
> Reloading exchange rates from file currency.xml
>    [junit4]   2> 541930 T733 oasc.CoreContainer.create Creating SolrCore
> 'collection1' using configuration from collection collection1
>    [junit4]   2> 541931 T733 oasc.SolrCore.initDirectoryFactory
> org.apache.solr.core.MockDirectoryFactory
>    [junit4]   2> 541931 T733 oasc.SolrCore.<init> [collection1] Opening
> new SolrCore at
> ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-003/collection1/,
> dataDir=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty1/
>    [junit4]   2> 541931 T733 oasc.JmxMonitoredMap.<init> JMX monitoring is
> enabled. Adding Solr mbeans to JMX Server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@6d87b6ba
>    [junit4]   2> 541932 T733 oasc.CachingDirectoryFactory.get return new
> directory for
> ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty1
>    [junit4]   2> 541932 T733 oasc.SolrCore.getNewIndexDir New index
> directory detected: old=null
> new=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty1/index/
>    [junit4]   2> 541933 T733 oasc.SolrCore.initIndex WARN [collection1]
> Solr index directory
> './solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty1/index'
> doesn't exist. Creating new index...
>    [junit4]   2> 541934 T733 oasc.CachingDirectoryFactory.get return new
> directory for
> ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty1/index
>    [junit4]   2> 541934 T733 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy:
> [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22,
> maxMergeSize=9223372036854775807,
> maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=true, maxMergeDocs=2147483647,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
>    [junit4]   2> 541935 T733 oasc.SolrDeletionPolicy.onCommit
> SolrDeletionPolicy.onCommit: commits: num=1
>    [junit4]   2>
>  commit{dir=BaseDirectoryWrapper(RAMDirectory@32aa968c
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@541c51e3
> ),segFN=segments_1,generation=1}
>    [junit4]   2> 541935 T733 oasc.SolrDeletionPolicy.updateCommits newest
> commit generation = 1
>    [junit4]   2> 541941 T733 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "nodistrib"
>    [junit4]   2> 541941 T733 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "dedupe"
>    [junit4]   2> 541942 T733 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "dedupe"
>    [junit4]   2> 541942 T733 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 541942 T733 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 541942 T733 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>    [junit4]   2> 541943 T733 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 541943 T733 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 541943 T733 oasc.SolrCore.loadUpdateProcessorChains no
> updateRequestProcessorChain defined as default, creating implicit default
>    [junit4]   2> 541944 T733 oasc.RequestHandlers.initHandlersFromConfig
> created /update: org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 541945 T733 oasc.RequestHandlers.initHandlersFromConfig
> created /update/json: org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 541945 T733 oasc.RequestHandlers.initHandlersFromConfig
> created /update/csv: org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 541946 T733 oasc.RequestHandlers.initHandlersFromConfig
> created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 541946 T733 oasc.RequestHandlers.initHandlersFromConfig
> created standard: solr.StandardRequestHandler
>    [junit4]   2> 541947 T733 oasc.RequestHandlers.initHandlersFromConfig
> created /get: solr.RealTimeGetHandler
>    [junit4]   2> 541947 T733 oasc.RequestHandlers.initHandlersFromConfig
> adding lazy requestHandler: solr.ReplicationHandler
>    [junit4]   2> 541947 T733 oasc.RequestHandlers.initHandlersFromConfig
> created /replication: solr.ReplicationHandler
>    [junit4]   2> 541948 T733 oasc.RequestHandlers.initHandlersFromConfig
> WARN Multiple requestHandler registered to the same name: /update ignoring:
> org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 541948 T733 oasc.RequestHandlers.initHandlersFromConfig
> created /update: solr.UpdateRequestHandler
>    [junit4]   2> 541949 T733 oasc.RequestHandlers.initHandlersFromConfig
> created /admin/: org.apache.solr.handler.admin.AdminHandlers
>    [junit4]   2> 541949 T733 oasc.RequestHandlers.initHandlersFromConfig
> created /admin/file: solr.admin.ShowFileRequestHandler
>    [junit4]   2> 541967 T733 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 541969 T733 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 541971 T733 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 541973 T733 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 541975 T733 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 541979 T733 oasu.CommitTracker.<init> Hard AutoCommit:
> disabled
>    [junit4]   2> 541979 T733 oasu.CommitTracker.<init> Soft AutoCommit:
> disabled
>    [junit4]   2> 541980 T733 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.MockRandomMergePolicy:
> org.apache.lucene.index.MockRandomMergePolicy@3f9e0f2d
>    [junit4]   2> 541981 T733 oasc.SolrDeletionPolicy.onInit
> SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>
>  commit{dir=BaseDirectoryWrapper(RAMDirectory@32aa968c
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@541c51e3
> ),segFN=segments_1,generation=1}
>    [junit4]   2> 541981 T733 oasc.SolrDeletionPolicy.updateCommits newest
> commit generation = 1
>    [junit4]   2> 541981 T733 oass.SolrIndexSearcher.<init> Opening
> Searcher@3b38b6cf[collection1] main
>    [junit4]   2> 541982 T733 oascc.ZkStateReader.readConfigName Load
> collection config from:/collections/collection1
>    [junit4]   2> 541982 T733 oascc.ZkStateReader.readConfigName
> path=/collections/collection1 configName=conf1 specified config exists in
> ZooKeeper
>    [junit4]   2> 541982 T733 oasr.ManagedResourceStorage.newStorageIO
> Setting up ZooKeeper-based storage for the RestManager with znodeBase:
> /configs/conf1
>    [junit4]   2> 541983 T733
> oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured
> ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 541983 T733 oasr.RestManager.init Initializing
> RestManager with initArgs: {}
>    [junit4]   2> 541983 T733 oasr.ManagedResourceStorage.load Reading
> _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 541984 T733
> oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream Read 38
> bytes from znode /configs/conf1/_rest_managed.json
>    [junit4]   2> 541984 T733 oasr.ManagedResourceStorage.load Loaded
> LinkedHashMap at path _rest_managed.json using
> ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 541985 T733 oasr.ManagedResource.reloadFromStorage Loaded
> initArgs {} for /rest/managed
>    [junit4]   2> 541985 T733 oasr.RestManager.init Initializing 0
> registered ManagedResources
>    [junit4]   2> 541989 T734 oasc.SolrCore.registerSearcher [collection1]
> Registered new searcher Searcher@3b38b6cf[collection1]
> main{StandardDirectoryReader(segments_1:1:nrt)}
>    [junit4]   2> 541993 T733 oasc.CoreContainer.registerCore registering
> core: collection1
>    [junit4]   2> 541993 T737 oasc.ZkController.register Register replica -
> core:collection1 address:http://127.0.0.1:25805/_h collection:collection1
> shard:shard1
>    [junit4]   2> 541994 T685 oass.SolrDispatchFilter.init
> user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
>    [junit4]   2> 541994 T685 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init() done
>    [junit4]   2> 541994 T737 oascc.SolrZkClient.makePath makePath:
> /collections/collection1/leader_elect/shard1/election
>    [junit4]   2> 541998 T737
> oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process
> for shard shard1
>    [junit4]   2> 542000 T711
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 542000 T737
> oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas
> found to continue.
>    [junit4]   2> 542000 T737
> oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader -
> try and sync
>    [junit4]   2> ASYNC  NEW_CORE C274 name=collection1
> org.apache.solr.core.SolrCore@22b0fea8 url=
> http://127.0.0.1:25805/_h/collection1 node=127.0.0.1:25805__h
> C274_STATE=coll:collection1 core:collection1 props:{state=down, base_url=
> http://127.0.0.1:25805/_h, core=collection1, node_name=127.0.0.1:25805__h}
>    [junit4]   2> 542001 T737 C274 P25805 oasc.SyncStrategy.sync Sync
> replicas to http://127.0.0.1:25805/_h/collection1/
>    [junit4]   2> 542001 T737 C274 P25805 oasc.SyncStrategy.syncReplicas
> Sync Success - now sync replicas to me
>    [junit4]   2> 542001 T708 oascc.ZkStateReader.updateClusterState
> Updating cloud state from ZooKeeper...
>    [junit4]   2> 542001 T737 C274 P25805 oasc.SyncStrategy.syncToMe
> http://127.0.0.1:25805/_h/collection1/ has no replicas
>    [junit4]   2> 542001 T737
> oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader:
> http://127.0.0.1:25805/_h/collection1/ shard1
>    [junit4]   2> 542002 T737 oascc.SolrZkClient.makePath makePath:
> /collections/collection1/leaders/shard1
>    [junit4]   2> 542003 T732 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 542003 T718 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 542003 T711 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 542006 T708 oascc.ZkStateReader.updateClusterState
> Updating cloud state from ZooKeeper...
>    [junit4]   2> 542008 T711
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 542008 T711 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 542008 T732 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 542008 T718 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 542056 T737 oasc.ZkController.register We are
> http://127.0.0.1:25805/_h/collection1/ and leader is
> http://127.0.0.1:25805/_h/collection1/
>    [junit4]   2> 542057 T737 oasc.ZkController.register No LogReplay
> needed for core=collection1 baseURL=http://127.0.0.1:25805/_h
>    [junit4]   2> 542057 T737 oasc.ZkController.checkRecovery I am the
> leader, no recovery necessary
>    [junit4]   2> 542057 T737 oasc.ZkController.publish publishing
> core=collection1 state=active collection=collection1
>    [junit4]   2> 542057 T737 oasc.ZkController.publish numShards not found
> on descriptor - reading it from system property
>    [junit4]   2> 542058 T711
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 542058 T737 oascc.ZkStateReader.updateClusterState
> Updating cloud state from ZooKeeper...
>    [junit4]   2> 542059 T708 oascc.ZkStateReader.updateClusterState
> Updating cloud state from ZooKeeper...
>    [junit4]   2> 542060 T708 oasc.Overseer$ClusterStateUpdater.updateState
> Update state numShards=3 message={
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "state":"active",
>    [junit4]   2>          "base_url":"http://127.0.0.1:25805/_h",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:25805__h",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "collection":"collection1",
>    [junit4]   2>          "numShards":"3",
>    [junit4]   2>          "core_node_name":"core_node1"}
>    [junit4]   2> 542163 T732 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 542163 T718 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 542163 T711 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
>    [junit4]   2> 543062 T685
> oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
>    [junit4]   2> 543063 T685 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 543065 T685 oejs.AbstractConnector.doStart Started
> SocketConnector@127.0.0.1:25808
>    [junit4]   2> 543066 T685 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()
>    [junit4]   2> 543066 T685 oasc.SolrResourceLoader.locateSolrHome JNDI
> not configured for solr (NoInitialContextEx)
>    [junit4]   2> 543066 T685 oasc.SolrResourceLoader.locateSolrHome using
> system property solr.solr.home:
> ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004
>    [junit4]   2> 543067 T685 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/'
>    [junit4]   2> 543096 T685 oasc.ConfigSolr.fromFile Loading container
> configuration from
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/solr.xml
>    [junit4]   2> 543146 T685 oasc.CoreContainer.<init> New CoreContainer
> 117677143
>    [junit4]   2> 543146 T685 oasc.CoreContainer.load Loading cores into
> CoreContainer
> [instanceDir=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/]
>    [junit4]   2> 543147 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 90000
>    [junit4]   2> 543147 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to:
>    [junit4]   2> 543147 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>    [junit4]   2> 543148 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>    [junit4]   2> 543148 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>    [junit4]   2> 543148 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>    [junit4]   2> 543148 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>    [junit4]   2> 543148 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>    [junit4]   2> 543149 T685 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>    [junit4]   2> 543149 T685 oasu.UpdateShardHandler.<init> Creating
> UpdateShardHandler HTTP client with params:
> socketTimeout=340000&connTimeout=45000&retry=false
>    [junit4]   2> 543149 T685 oasl.LogWatcher.createWatcher SLF4J impl is
> org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 543150 T685 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 543150 T685 oasc.CoreContainer.load Host Name: 127.0.0.1
>    [junit4]   2> 543150 T685 oasc.ZkContainer.initZooKeeper Zookeeper
> client=127.0.0.1:25798/solr
>    [junit4]   2> 543150 T685 oasc.ZkController.checkChrootPath zkHost
> includes chroot
>    [junit4]   2> 543151 T685 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 543153 T748 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@7e0d1759
> name:ZooKeeperConnection Watcher:127.0.0.1:25798 got event WatchedEvent
> state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 543154 T685 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 543156 T685 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 543157 T750 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@18e085c9
> name:ZooKeeperConnection Watcher:127.0.0.1:25798/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 543157 T685 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 543161 T685
> oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
> state from ZooKeeper...
>    [junit4]   2> 544164 T685 oasc.ZkController.createEphemeralLiveNode
> Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25808__h
>    [junit4]   2> 544165 T685 oascc.SolrZkClient.makePath makePath:
> /live_nodes/127.0.0.1:25808__h
>    [junit4]   2> 544168 T711 oascc.ZkStateReader$3.process Updating live
> nodes... (3)
>    [junit4]   2> 544168 T732 oascc.ZkStateReader$3.process Updating live
> nodes... (3)
>    [junit4]   2> 544168 T751 oascc.ZkStateReader$3.process Updating live
> nodes... (3)
>    [junit4]   2> 544168 T718 oascc.ZkStateReader$3.process Updating live
> nodes... (3)
>    [junit4]   2> 544174 T752 oasc.ZkController.publish publishing
> core=collection1 state=down collection=collection1
>    [junit4]   2> 544175 T752 oasc.ZkController.publish numShards not found
> on descriptor - reading it from system property
>    [junit4]   2> 544175 T752 oasc.ZkController.waitForCoreNodeName look
> for our core node name
>    [junit4]   2> 544175 T711
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>    [junit4]   2> 544176 T708 oascc.ZkStateReader.updateClusterState
> Updating cloud state from ZooKeeper...
>    [junit4]   2> 544177 T708 oasc.Overseer$ClusterStateUpdater.updateState
> Update state numShards=3 message={
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "base_url":"http://127.0.0.1:25808/_h",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:25808__h",
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "collection":"collection1",
>    [junit4]   2>          "numShards":"3",
>    [junit4]   2>          "core_node_name":null}
>    [junit4]   2> 544177 T708 oasc.Overseer$ClusterStateUpdater.updateState
> Collection already exists with numShards=3
>    [junit4]   2> 544177 T708 oasc.Overseer$ClusterStateUpdater.updateState
> Assigning new node to shard shard=shard2
>    [junit4]   2> 544179 T751 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
>    [junit4]   2> 544179 T711 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
>    [junit4]   2> 544179 T718 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
>    [junit4]   2> 544179 T732 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
>    [junit4]   2> 545176 T752 oasc.ZkController.waitForShardId waiting to
> find shard id in clusterstate for collection1
>    [junit4]   2> 545177 T752 oasc.ZkController.createCollectionZkNode
> Check for collection zkNode:collection1
>    [junit4]   2> 545177 T752 oasc.ZkController.createCollectionZkNode
> Collection zkNode exists
>    [junit4]   2> 545177 T752 oascc.ZkStateReader.readConfigName Load
> collection config from:/collections/collection1
>    [junit4]   2> 545178 T752 oascc.ZkStateReader.readConfigName
> path=/collections/collection1 configName=conf1 specified config exists in
> ZooKeeper
>    [junit4]   2> 545178 T752 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/collection1/'
>    [junit4]   2> 545179 T752 oasc.SolrResourceLoader.replaceClassLoader
> Adding
> 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/collection1/lib/.svn/'
> to classloader
>    [junit4]   2> 545180 T752 oasc.SolrResourceLoader.replaceClassLoader
> Adding
> 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/collection1/lib/README'
> to classloader
>    [junit4]   2> 545180 T752 oasc.SolrResourceLoader.replaceClassLoader
> Adding
> 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/collection1/lib/classes/'
> to classloader
>    [junit4]   2> 545223 T752 oasc.SolrConfig.<init> Using Lucene
> MatchVersion: LUCENE_4_10
>    [junit4]   2> 545254 T752 oasc.SolrConfig.<init> Loaded SolrConfig:
> solrconfig.xml
>    [junit4]   2> 545255 T752 oass.IndexSchema.readSchema Reading Solr
> Schema from schema.xml
>    [junit4]   2> 545261 T752 oass.IndexSchema.readSchema [collection1]
> Schema name=test
>    [junit4]   2> 545512 T752 oass.IndexSchema.readSchema default search
> field in schema is text
>    [junit4]   2> 545514 T752 oass.IndexSchema.readSchema unique key field:
> id
>    [junit4]   2> 545516 T752 oass.FileExchangeRateProvider.reload
> Reloading exchange rates from file currency.xml
>    [junit4]   2> 545519 T752 oass.FileExchangeRateProvider.reload
> Reloading exchange rates from file currency.xml
>    [junit4]   2> 545543 T752 oasc.CoreContainer.create Creating SolrCore
> 'collection1' using configuration from collection collection1
>    [junit4]   2> 545543 T752 oasc.SolrCore.initDirectoryFactory
> org.apache.solr.core.MockDirectoryFactory
>    [junit4]   2> 545543 T752 oasc.SolrCore.<init> [collection1] Opening
> new SolrCore at
> ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-004/collection1/,
> dataDir=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty2/
>    [junit4]   2> 545544 T752 oasc.JmxMonitoredMap.<init> JMX monitoring is
> enabled. Adding Solr mbeans to JMX Server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@6d87b6ba
>    [junit4]   2> 545545 T752 oasc.CachingDirectoryFactory.get return new
> directory for
> ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty2
>    [junit4]   2> 545545 T752 oasc.SolrCore.getNewIndexDir New index
> directory detected: old=null
> new=./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty2/index/
>    [junit4]   2> 545546 T752 oasc.SolrCore.initIndex WARN [collection1]
> Solr index directory
> './solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty2/index'
> doesn't exist. Creating new index...
>    [junit4]   2> 545546 T752 oasc.CachingDirectoryFactory.get return new
> directory for
> ./solr.cloud.FullSolrCloudDistribCmdsTest-E7864240DC5F2611-001/tempDir-001/jetty2/index
>    [junit4]   2> 545547 T752 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy:
> [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22,
> maxMergeSize=9223372036854775807,
> maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=true, maxMergeDocs=2147483647,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
>    [junit4]   2> 545548 T752 oasc.SolrDeletionPolicy.onCommit
> SolrDeletionPolicy.onCommit: commits: num=1
>    [junit4]   2>
>  commit{dir=BaseDirectoryWrapper(RAMDirectory@3d44ce56
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@17ba34c2
> ),segFN=segments_1,generation=1}
>    [junit4]   2> 545548 T752 oasc.SolrDeletionPolicy.updateCommits newest
> commit generation = 1
>    [junit4]   2> 545553 T752 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "nodistrib"
>    [junit4]   2> 545554 T752 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "dedupe"
>    [junit4]   2> 545554 T752 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "dedupe"
>    [junit4]   2> 545554 T752 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 545555 T752 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 545555 T752 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>    [junit4]   2> 545555 T752 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 545556 T752 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 545556 T752 oasc.SolrCore.loadUpdateProcessorChains no
> updateRequestProcessorChain defined as default, creating implicit default
>    [junit4]   2> 545557 T752 oasc.RequestHandlers.initHandlersFromConfig
> created /update: org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 545557 T752 oasc.RequestHandlers.initHandlersFromConfig
> created /update/json: org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 545558 T752 oasc.RequestHandlers.initHandlersFromConfig
> created /update/csv: org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 545558 T752 oasc.RequestHandlers.initHandlersFromConfig
> created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 545559 T752 oasc.RequestHandlers.initHandlersFromConfig
> created standard: solr.StandardRequestHandler
>    [junit4]   2> 545559 T752 oasc.RequestHandlers.initHandlersFromConfig
> created /get: solr.RealTimeGetHandler
>    [junit4]   2> 545560 T752 oasc.RequestHandlers.initHandlersFromConfig
> adding lazy requestHandler: solr.ReplicationHandler
>    [junit4]   2> 545560 T752 oasc.RequestHandlers.initHandlersFromConfig
> created /replication: solr.ReplicationHandler
>    [junit4]   2> 545561 T752 oasc.RequestHandlers.initHandlersFromConfig
> WARN Multiple requestHandler registered to the same name: /update ignoring:
> org.apache.solr.handler.UpdateRequestHandler
>    [junit4]   2> 545561 T752 oasc.RequestHandlers.initHandlersFromConfig
> created /update: solr.UpdateRequestHandler
>    [junit4]   2> 545561 T752 oasc.RequestHandlers.initHandlersFromConfig
> created /admin/: org.apache.solr.handler.admin.AdminHandlers
>    [junit4]   2> 545562 T752 oasc.RequestHandlers.initHandlersFromConfig
> created /admin/file: solr.admin.ShowFileRequestHandler
>    [junit4]   2> 545580 T752 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 545582 T752 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 545584 T752 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 545586 T752 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 545588 T752 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>    [junit4]   2> 545592 T752 oasu.CommitTracker.<init> Hard AutoCommit:
> disabled
>    [junit4]   2> 545592 T752 oasu.CommitTracker.<init> Soft AutoCommit:
> disabled
>    [junit4]   2> 545593 T752 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.MockRandomMergePolicy:
> org.apache.lucene.index.MockRandomMergePolicy@612d1e09
>    [junit4]   2> 545594 T752 oasc.SolrDeletionPolicy.onInit
> SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>
>  commit{dir=BaseDirectoryWrapper(RAMDirectory@3d44ce56
> lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@17ba34c2
> ),segFN=segments_1,generation=1}
>    [junit4]   2> 545594 T752 oasc.SolrDeletionPolicy.updateCommits newest
> commit generation = 1
>    [junit4]   2> 5
>
> [...truncated too long message...]
>
> elds
>    [junit4]   2> 5422859 T31606 oas.SolrTestCaseJ4.deleteCore ###deleteCore
>    [junit4]   2> 5422859 T31606 oasc.CoreContainer.shutdown Shutting down
> CoreContainer instance=1384115479
>    [junit4]   2> 5422859 T31606 oasc.SolrCore.close [collection1]  CLOSING
> SolrCore org.apache.solr.core.SolrCore@510aa80f
>    [junit4]   2> 5422860 T31606 oasu.DirectUpdateHandler2.close closing
> DirectUpdateHandler2{commits=443,autocommits=0,soft
> autocommits=0,optimizes=17,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=20227,cumulative_deletesById=155,cumulative_deletesByQuery=18,cumulative_errors=0}
>    [junit4]   2> 5422860 T31606 oasu.SolrCoreState.decrefSolrCoreState
> Closing SolrCoreState
>    [junit4]   2> 5422860 T31606 oasu.DefaultSolrCoreState.closeIndexWriter
> SolrCoreState ref count has reached 0 - closing IndexWriter
>    [junit4]   2> 5422860 T31606 oasu.DefaultSolrCoreState.closeIndexWriter
> closing IndexWriter with IndexWriterCloser
>    [junit4]   2> 5422860 T31606 oasc.SolrCore.closeSearcher [collection1]
> Closing main searcher on request.
>    [junit4]   2> 5422861 T31606 oasc.CachingDirectoryFactory.close Closing
> MockDirectoryFactory - 2 directories currently being tracked
>    [junit4]   2> 5422861 T31606
> oasc.CachingDirectoryFactory.closeCacheValue looking to close
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.request.TestIntervalFaceting-E7864240DC5F2611-001/init-core-data-001/index
> [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.request.TestIntervalFaceting-E7864240DC5F2611-001/init-core-data-001/index;done=false>>]
>    [junit4]   2> 5422861 T31606 oasc.CachingDirectoryFactory.close Closing
> directory:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.request.TestIntervalFaceting-E7864240DC5F2611-001/init-core-data-001/index
>    [junit4]   2> 5422861 T31606
> oasc.CachingDirectoryFactory.closeCacheValue looking to close
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.request.TestIntervalFaceting-E7864240DC5F2611-001/init-core-data-001
> [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.request.TestIntervalFaceting-E7864240DC5F2611-001/init-core-data-001;done=false>>]
>    [junit4]   2> 5422862 T31606 oasc.CachingDirectoryFactory.close Closing
> directory:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.request.TestIntervalFaceting-E7864240DC5F2611-001/init-core-data-001
>    [junit4]   2> NOTE: leaving temporary files on disk at:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.request.TestIntervalFaceting-E7864240DC5F2611-001
>    [junit4]   2> NOTE: test params are: codec=Lucene49: {test_l_dv=FST41,
> test_d_dv=FST41, test_f_dv=PostingsFormat(name=FSTPulsing41),
> test_fs_dv=Pulsing41(freqCutoff=15 minBlockSize=44 maxBlockSize=113),
> test_dt_dv=FST41, test_ls_dv=PostingsFormat(name=NestedPulsing),
> test_ds_dv=PostingsFormat(name=NestedPulsing), test_s=FST41,
> id=Pulsing41(freqCutoff=15 minBlockSize=44 maxBlockSize=113)},
> docValues:{test_l_dv=DocValuesFormat(name=Lucene49),
> test_d_dv=DocValuesFormat(name=Lucene49),
> test_is_dv=DocValuesFormat(name=Asserting),
> test_f_dv=DocValuesFormat(name=Asserting),
> test_fs_dv=DocValuesFormat(name=SimpleText),
> id_dv=DocValuesFormat(name=Asserting),
> test_ss_dv=DocValuesFormat(name=Lucene49),
> test_dt_dv=DocValuesFormat(name=Lucene49),
> test_ls_dv=DocValuesFormat(name=Memory),
> test_ds_dv=DocValuesFormat(name=Memory),
> test_s_dv=DocValuesFormat(name=SimpleText),
> test_i_dv=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity,
> locale=th_TH_TH_#u-nu-thai, timezone=Asia/Dacca
>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation
> 1.7.0_60 (64-bit)/cpus=16,threads=1,free=175281744,total=536346624
>    [junit4]   2> NOTE: All tests run in this JVM: [TestCoreContainer,
> CoreContainerCoreInitFailuresTest, SolrRequestParserTest,
> TestBlendedInfixSuggestions, TestIndexSearcher, ResponseHeaderTest,
> BJQParserTest, TestLRUCache, QueryParsingTest, LukeRequestHandlerTest,
> CurrencyFieldXmlFileTest, TestCloudInspectUtil, TestMergePolicyConfig,
> TestValueSourceCache, ShardSplitTest, TestRecovery, TestNonNRTOpen,
> TestManagedStopFilterFactory, TestLFUCache, SliceStateTest,
> FullSolrCloudDistribCmdsTest, HdfsBasicDistributedZkTest,
> HighlighterMaxOffsetTest, SolrInfoMBeanTest, HdfsLockFactoryTest,
> TestPerFieldSimilarity, OverseerRolesTest, UUIDFieldTest,
> TestQuerySenderNoQuery, TestZkChroot, TestReversedWildcardFilterFactory,
> TestQueryUtils, URLClassifyProcessorTest, TestImplicitCoreProperties,
> AnalyticsMergeStrategyTest, SortByFunctionTest, TestSchemaNameResource,
> SystemInfoHandlerTest, InfoHandlerTest, TestSolrXmlPersistor,
> TestLeaderElectionZkExpiry, FieldAnalysisRequestHandlerTest,
> TestMissingGroups, DirectSolrConnectionTest, TestFiltering,
> DirectUpdateHandlerOptimizeTest, EnumFieldTest, BlockCacheTest,
> LoggingHandlerTest, TestPHPSerializedResponseWriter, TestJoin,
> TestDynamicFieldCollectionResource, DOMUtilTest, AlternateDirectoryTest,
> TestDocBasedVersionConstraints, OutputWriterTest, TestFastWriter,
> DebugComponentTest, TestManagedSchemaFieldResource,
> TestSweetSpotSimilarityFactory, SuggestComponentTest, MultiTermTest,
> TestExtendedDismaxParser, DocValuesTest, AutoCommitTest, TestFastLRUCache,
> TestBinaryField, TestCSVResponseWriter, TestLazyCores,
> DistributedQueryElevationComponentTest, CustomCollectionTest,
> ShardRoutingCustomTest, TestSystemIdResolver, TestAddFieldRealTimeGet,
> TestShardHandlerFactory, SpatialFilterTest, TestStressLucene,
> CSVRequestHandlerTest, DistributedQueryComponentCustomSortTest,
> TestGroupingSearch, TestDocumentBuilder, TestCopyFieldCollectionResource,
> RequestHandlersTest, CircularListTest, TestPhraseSuggestions, SampleTest,
> TestCSVLoader, TestPseudoReturnFields, PrimUtilsTest, TestSolr4Spatial2,
> TestUpdate, AddBlockUpdateTest, TestAtomicUpdateErrorCases,
> SolrIndexConfigTest, BasicZkTest, UniqFieldsUpdateProcessorFactoryTest,
> ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test,
> UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, SyncSliceTest,
> OpenCloseCoreStressTest, LeaderElectionIntegrationTest, RecoveryZkTest,
> LeaderElectionTest, ZkCLITest, TestFaceting,
> TermVectorComponentDistributedTest, TestRealTimeGet,
> TestMultiCoreConfBootstrap, HardAutoCommitTest,
> DistributedTermsComponentTest, QueryElevationComponentTest,
> BadIndexSchemaTest, ConvertedLegacyTest, TestFunctionQuery,
> DirectUpdateHandlerTest, SoftAutoCommitTest, CurrencyFieldOpenExchangeTest,
> SimplePostToolTest, TestCoreDiscovery, SuggesterFSTTest,
> CoreAdminHandlerTest, PolyFieldTest, SchemaVersionSpecificBehaviorTest,
> FieldMutatingUpdateProcessorTest, DefaultValueUpdateProcessorTest,
> DocValuesMultiTest, DistanceFunctionTest, CacheHeaderTest, TestWriterPerf,
> DocumentAnalysisRequestHandlerTest, TestOmitPositions, TermsComponentTest,
> PathHierarchyTokenizerFactoryTest, TestIndexingPerformance,
> RequiredFieldsTest, FastVectorHighlighterTest, RegexBoostProcessorTest,
> TestCollationField, UpdateRequestProcessorFactoryTest,
> TestAnalyzedSuggestions, TestPartialUpdateDeduplication,
> TestBinaryResponseWriter, TestSolrIndexConfig, ResponseLogComponentTest,
> TestStressRecovery, PreAnalyzedFieldTest, DateFieldTest,
> SpellingQueryConverterTest, ClusterStateTest, TestUtils, FileUtilsTest,
> CursorPagingTest, DistributedIntervalFacetingTest, ConnectionManagerTest,
> DeleteInactiveReplicaTest, DistribCursorPagingTest, SSLMigrationTest,
> SolrXmlInZkTest, TestDistribDocBasedVersion, TestMiniSolrCloudCluster,
> TestRequestStatusCollectionAPI, TestShortCircuitedRequests,
> HdfsChaosMonkeySafeLeaderTest, HdfsRecoveryZkTest,
> HdfsUnloadDistributedZkTest, CachingDirectoryFactoryTest,
> QueryResultKeyTest, TestConfigSets, TestInfoStreamLogging,
> TestReloadAndDeleteDocs, CoreAdminCreateDiscoverTest,
> CoreAdminRequestStatusTest, CoreMergeIndexesAdminHandlerTest,
> DistributedDebugComponentTest, DistributedQueryComponentOptimizationTest,
> DistributedSuggestComponentTest, TestIntervalFaceting]
>    [junit4] Completed on J1 in 197.51s, 17 tests, 3 failures <<< FAILURES!
>
> [...truncated 162 lines...]
> BUILD FAILED
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:481:
> The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:454:
> The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:45:
> The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/extra-targets.xml:37:
> The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build.xml:189:
> The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/common-build.xml:496:
> The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1308:
> The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:923:
> There were test failures: 410 suites, 1711 tests, 1 error, 3 failures, 50
> ignored (8 assumptions)
>
> Total time: 281 minutes 47 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-NightlyTests-4.x #586
> Archived 3 artifacts
> Archive block size is 32768
> Received 0 blocks and 462708180 bytes
> Compression is 0.0%
> Took 1 min 30 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>