You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2014/07/03 04:45:26 UTC

[JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 569 - Still Failing

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/569/

2 tests failed.
REGRESSION:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:17323/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17323/collection1
	at __randomizedtesting.SeedInfo.seed([EFDCB4F4D6D1BBF:8F1B45573A327B83]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:223)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:165)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrServer(FullSolrCloudDistribCmdsTest.java:415)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.doTest(FullSolrCloudDistribCmdsTest.java:145)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketException: Broken pipe
	at java.net.SocketOutputStream.socketWrite0(Native Method)
	at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113)
	at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
	at org.apache.http.impl.io.AbstractSessionOutputBuffer.flushBuffer(AbstractSessionOutputBuffer.java:159)
	at org.apache.http.impl.io.AbstractSessionOutputBuffer.writeLine(AbstractSessionOutputBuffer.java:258)
	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:59)
	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:46)
	at org.apache.http.impl.io.AbstractMessageWriter.write(AbstractMessageWriter.java:109)
	at org.apache.http.impl.AbstractHttpClientConnection.sendRequestHeader(AbstractHttpClientConnection.java:254)
	at org.apache.http.impl.conn.DefaultClientConnection.sendRequestHeader(DefaultClientConnection.java:270)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.sendRequestHeader(ManagedClientConnectionImpl.java:209)
	at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:203)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:121)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:452)
	... 50 more


REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.testDistribSearch

Error Message:
shard1 is not consistent.  Got 78 from http://127.0.0.1:50417/x_/h/collection1lastClient and got 46 from http://127.0.0.1:54345/x_/h/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 78 from http://127.0.0.1:50417/x_/h/collection1lastClient and got 46 from http://127.0.0.1:54345/x_/h/collection1
	at __randomizedtesting.SeedInfo.seed([EFDCB4F4D6D1BBF:8F1B45573A327B83]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.doTest(RecoveryZkTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 12007 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryZkTest
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/init-core-data-001
   [junit4]   2> 3000754 T455281 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true)
   [junit4]   2> 3000755 T455281 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /x_/h
   [junit4]   2> 3000761 T455281 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 3000762 T455281 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3000763 T455282 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 3000863 T455281 oasc.ZkTestServer.run start zk server on port:15150
   [junit4]   2> 3000865 T455281 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3000873 T455288 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6be3d3a0 name:ZooKeeperConnection Watcher:127.0.0.1:15150 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3000873 T455281 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3000874 T455281 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 3000888 T455281 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3000890 T455290 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4019eb name:ZooKeeperConnection Watcher:127.0.0.1:15150/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3000890 T455281 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3000891 T455281 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 3000895 T455281 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 3000905 T455281 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 3000907 T455281 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 3000909 T455281 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 3000910 T455281 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 3000926 T455281 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 3000927 T455281 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 3001030 T455281 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 3001031 T455281 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 3001034 T455281 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 3001034 T455281 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 3001042 T455281 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 3001043 T455281 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 3001045 T455281 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 3001045 T455281 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 3001048 T455281 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 3001048 T455281 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 3001050 T455281 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 3001051 T455281 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 3001059 T455281 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 3001059 T455281 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 3001061 T455281 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 3001062 T455281 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 3001065 T455281 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 3001065 T455281 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 3002138 T455281 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 3002148 T455281 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:50360
   [junit4]   2> 3002148 T455281 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 3002148 T455281 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 3002149 T455281 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-002
   [junit4]   2> 3002149 T455281 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-002/'
   [junit4]   2> 3002178 T455281 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-002/solr.xml
   [junit4]   2> 3002227 T455281 oasc.CoreContainer.<init> New CoreContainer 786566782
   [junit4]   2> 3002228 T455281 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-002/]
   [junit4]   2> 3002229 T455281 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 3002229 T455281 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 3002229 T455281 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 3002229 T455281 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 3002230 T455281 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 3002230 T455281 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 3002230 T455281 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 3002230 T455281 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 3002231 T455281 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 3002231 T455281 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 3002231 T455281 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 3002232 T455281 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 3002232 T455281 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:15150/solr
   [junit4]   2> 3002232 T455281 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 3002233 T455281 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3002235 T455301 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b258a3d name:ZooKeeperConnection Watcher:127.0.0.1:15150 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3002238 T455281 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3002252 T455281 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3002253 T455303 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25d66866 name:ZooKeeperConnection Watcher:127.0.0.1:15150/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3002254 T455281 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3002257 T455281 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 3002266 T455281 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 3002274 T455281 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 3002284 T455281 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 3002292 T455281 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 3002294 T455281 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 3002297 T455281 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50360_x_%2Fh
   [junit4]   2> 3002305 T455281 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50360_x_%2Fh
   [junit4]   2> 3002308 T455281 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 3002310 T455281 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 3002323 T455281 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:50360_x_%2Fh
   [junit4]   2> 3002323 T455281 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 3002324 T455281 oasc.Overseer.start Overseer (id=92035692780781571-127.0.0.1:50360_x_%2Fh-n_0000000000) starting
   [junit4]   2> 3002335 T455281 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 3002349 T455305 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 3002349 T455281 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 3002354 T455281 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 3002355 T455281 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3002356 T455304 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 3002360 T455306 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 3002360 T455306 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 3002361 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3002361 T455306 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 3002362 T455304 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3002362 T455304 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50360/x_/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50360_x_%2Fh",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 3002363 T455304 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 3002363 T455304 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 3002365 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3002366 T455303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3003362 T455306 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 3003363 T455306 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 3003363 T455306 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 3003364 T455306 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 3003364 T455306 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 3003364 T455306 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-002/collection1/'
   [junit4]   2> 3003366 T455306 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 3003367 T455306 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 3003368 T455306 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 3003433 T455306 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 3003489 T455306 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 3003591 T455306 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 3003600 T455306 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 3003955 T455306 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 3003958 T455306 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 3003960 T455306 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 3003964 T455306 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 3003977 T455306 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 3003977 T455306 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 3003977 T455306 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-002/collection1/, dataDir=./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/control/data/
   [junit4]   2> 3003978 T455306 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@61871708
   [junit4]   2> 3003978 T455306 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/control/data
   [junit4]   2> 3003979 T455306 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/control/data/index/
   [junit4]   2> 3003979 T455306 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 3003979 T455306 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/control/data/index
   [junit4]   2> 3003980 T455306 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=5.2294921875, floorSegmentMB=0.6162109375, forceMergeDeletesPctAllowed=26.593516805814566, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 3003981 T455306 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5d397694 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2b9dd26e),segFN=segments_1,generation=1}
   [junit4]   2> 3003981 T455306 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 3003983 T455306 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 3003984 T455306 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 3003984 T455306 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 3003984 T455306 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 3003984 T455306 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 3003985 T455306 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 3003985 T455306 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 3003985 T455306 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 3003985 T455306 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 3003986 T455306 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 3003986 T455306 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 3003986 T455306 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 3003987 T455306 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 3003987 T455306 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 3003987 T455306 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 3003988 T455306 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 3004007 T455306 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 3004012 T455306 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 3004012 T455306 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 3004013 T455306 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=10.1904296875, floorSegmentMB=1.755859375, forceMergeDeletesPctAllowed=26.26056659028575, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.767914023905204
   [junit4]   2> 3004013 T455306 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5d397694 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2b9dd26e),segFN=segments_1,generation=1}
   [junit4]   2> 3004014 T455306 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 3004014 T455306 oass.SolrIndexSearcher.<init> Opening Searcher@1c86dc30[collection1] main
   [junit4]   2> 3004014 T455306 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 3004015 T455306 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 3004015 T455306 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 3004015 T455306 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 3004016 T455306 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 3004016 T455306 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 3004016 T455306 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 3004017 T455306 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 3004017 T455306 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 3004017 T455306 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 3004019 T455307 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1c86dc30[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 3004020 T455306 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 3004020 T455310 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50360/x_/h collection:control_collection shard:shard1
   [junit4]   2> 3004020 T455281 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
   [junit4]   2> 3004021 T455281 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 3004021 T455310 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 3004022 T455281 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3004024 T455312 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a739590 name:ZooKeeperConnection Watcher:127.0.0.1:15150/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3004024 T455281 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3004036 T455281 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3004039 T455310 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 3004039 T455281 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 3004040 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3004041 T455310 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 3004041 T455310 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C22748 name=collection1 org.apache.solr.core.SolrCore@730af28d url=http://127.0.0.1:50360/x_/h/collection1 node=127.0.0.1:50360_x_%2Fh C22748_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:50360/x_/h, core=collection1, node_name=127.0.0.1:50360_x_%2Fh}
   [junit4]   2> 3004041 T455310 C22748 P50360 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50360/x_/h/collection1/
   [junit4]   2> 3004041 T455304 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3004042 T455310 C22748 P50360 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 3004042 T455310 C22748 P50360 oasc.SyncStrategy.syncToMe http://127.0.0.1:50360/x_/h/collection1/ has no replicas
   [junit4]   2> 3004042 T455310 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50360/x_/h/collection1/ shard1
   [junit4]   2> 3004043 T455310 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 3004044 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3004044 T455303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3004044 T455312 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3004049 T455304 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3004051 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3004154 T455303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3004154 T455312 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3004201 T455310 oasc.ZkController.register We are http://127.0.0.1:50360/x_/h/collection1/ and leader is http://127.0.0.1:50360/x_/h/collection1/
   [junit4]   2> 3004202 T455310 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50360/x_/h
   [junit4]   2> 3004202 T455310 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 3004202 T455310 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 3004202 T455310 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 3004204 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3004204 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3004204 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3004204 T455310 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3004205 T455304 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3004205 T455304 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50360/x_/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50360_x_%2Fh",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 3004215 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3004318 T455312 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3004318 T455303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 3005145 T455281 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 3005146 T455281 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 3005149 T455281 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:50417
   [junit4]   2> 3005150 T455281 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 3005150 T455281 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 3005150 T455281 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-003
   [junit4]   2> 3005151 T455281 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-003/'
   [junit4]   2> 3005180 T455281 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-003/solr.xml
   [junit4]   2> 3005231 T455281 oasc.CoreContainer.<init> New CoreContainer 1583061663
   [junit4]   2> 3005231 T455281 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-003/]
   [junit4]   2> 3005232 T455281 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 3005232 T455281 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 3005233 T455281 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 3005233 T455281 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 3005233 T455281 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 3005233 T455281 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 3005233 T455281 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 3005234 T455281 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 3005234 T455281 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 3005234 T455281 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 3005235 T455281 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 3005235 T455281 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 3005235 T455281 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:15150/solr
   [junit4]   2> 3005235 T455281 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 3005236 T455281 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3005238 T455323 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@50046d1e name:ZooKeeperConnection Watcher:127.0.0.1:15150 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3005241 T455281 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3005257 T455281 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3005259 T455325 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60ae6bbc name:ZooKeeperConnection Watcher:127.0.0.1:15150/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3005259 T455281 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3005272 T455281 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3006275 T455281 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50417_x_%2Fh
   [junit4]   2> 3006305 T455281 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50417_x_%2Fh
   [junit4]   2> 3006309 T455303 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3006309 T455312 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3006309 T455325 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 3006322 T455326 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 3006322 T455326 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 3006323 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3006323 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3006323 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3006323 T455326 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 3006324 T455304 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3006325 T455304 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50417/x_/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50417_x_%2Fh",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 3006325 T455304 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
   [junit4]   2> 3006325 T455304 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 3006333 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3006334 T455325 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3006334 T455312 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3006334 T455303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3007324 T455326 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 3007325 T455326 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 3007325 T455326 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 3007326 T455326 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 3007326 T455326 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 3007326 T455326 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-003/collection1/'
   [junit4]   2> 3007328 T455326 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 3007328 T455326 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 3007329 T455326 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 3007402 T455326 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 3007433 T455326 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 3007535 T455326 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 3007541 T455326 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 3007838 T455326 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 3007841 T455326 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 3007843 T455326 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 3007847 T455326 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 3007861 T455326 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 3007862 T455326 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 3007862 T455326 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-003/collection1/, dataDir=./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/jetty1/
   [junit4]   2> 3007862 T455326 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@61871708
   [junit4]   2> 3007863 T455326 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/jetty1
   [junit4]   2> 3007863 T455326 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/jetty1/index/
   [junit4]   2> 3007864 T455326 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 3007866 T455326 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/jetty1/index
   [junit4]   2> 3007867 T455326 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=5.2294921875, floorSegmentMB=0.6162109375, forceMergeDeletesPctAllowed=26.593516805814566, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 3007868 T455326 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4f55ad70 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7295da4),segFN=segments_1,generation=1}
   [junit4]   2> 3007868 T455326 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 3007871 T455326 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 3007871 T455326 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 3007871 T455326 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 3007872 T455326 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 3007872 T455326 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 3007872 T455326 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 3007873 T455326 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 3007873 T455326 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 3007873 T455326 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 3007874 T455326 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 3007874 T455326 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 3007874 T455326 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 3007874 T455326 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 3007875 T455326 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 3007875 T455326 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 3007876 T455326 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 3007894 T455326 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 3007899 T455326 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 3007899 T455326 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 3007900 T455326 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=10.1904296875, floorSegmentMB=1.755859375, forceMergeDeletesPctAllowed=26.26056659028575, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.767914023905204
   [junit4]   2> 3007900 T455326 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4f55ad70 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7295da4),segFN=segments_1,generation=1}
   [junit4]   2> 3007901 T455326 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 3007901 T455326 oass.SolrIndexSearcher.<init> Opening Searcher@2803f8f8[collection1] main
   [junit4]   2> 3007901 T455326 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 3007902 T455326 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 3007902 T455326 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 3007903 T455326 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 3007903 T455326 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 3007903 T455326 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 3007903 T455326 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 3007904 T455326 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 3007904 T455326 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 3007904 T455326 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 3007907 T455327 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2803f8f8[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 3007909 T455326 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 3007909 T455330 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50417/x_/h collection:collection1 shard:shard1
   [junit4]   2> 3007909 T455281 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
   [junit4]   2> 3007910 T455330 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 3007910 T455281 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 3007927 T455330 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 3007938 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3007938 T455330 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 3007938 T455330 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C22749 name=collection1 org.apache.solr.core.SolrCore@7d56b881 url=http://127.0.0.1:50417/x_/h/collection1 node=127.0.0.1:50417_x_%2Fh C22749_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:50417/x_/h, core=collection1, node_name=127.0.0.1:50417_x_%2Fh}
   [junit4]   2> 3007939 T455330 C22749 P50417 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50417/x_/h/collection1/
   [junit4]   2> 3007939 T455304 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3007939 T455330 C22749 P50417 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 3007939 T455330 C22749 P50417 oasc.SyncStrategy.syncToMe http://127.0.0.1:50417/x_/h/collection1/ has no replicas
   [junit4]   2> 3007940 T455330 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50417/x_/h/collection1/ shard1
   [junit4]   2> 3007940 T455330 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 3007941 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3007950 T455303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3007951 T455325 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3007951 T455312 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3007962 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3007963 T455304 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3007965 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3008068 T455303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3008068 T455312 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3008068 T455325 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3008115 T455330 oasc.ZkController.register We are http://127.0.0.1:50417/x_/h/collection1/ and leader is http://127.0.0.1:50417/x_/h/collection1/
   [junit4]   2> 3008116 T455330 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50417/x_/h
   [junit4]   2> 3008116 T455330 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 3008116 T455330 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 3008116 T455330 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 3008118 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3008118 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3008118 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3008118 T455330 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3008119 T455304 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3008119 T455304 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50417/x_/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50417_x_%2Fh",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 3008129 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3008232 T455303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3008232 T455312 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3008232 T455325 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 3008867 T455281 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 3008868 T455281 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 3008873 T455281 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54345
   [junit4]   2> 3008874 T455281 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 3008874 T455281 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 3008874 T455281 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-004
   [junit4]   2> 3008874 T455281 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-004/'
   [junit4]   2> 3008903 T455281 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-004/solr.xml
   [junit4]   2> 3008953 T455281 oasc.CoreContainer.<init> New CoreContainer 1251860337
   [junit4]   2> 3008953 T455281 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-004/]
   [junit4]   2> 3008966 T455281 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 3008967 T455281 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 3008967 T455281 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 3008967 T455281 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 3008968 T455281 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 3008968 T455281 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 3008968 T455281 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 3008968 T455281 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 3008968 T455281 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 3008969 T455281 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 3008969 T455281 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 3008969 T455281 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 3008970 T455281 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:15150/solr
   [junit4]   2> 3008970 T455281 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 3008971 T455281 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3008973 T455341 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e06943e name:ZooKeeperConnection Watcher:127.0.0.1:15150 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3008975 T455281 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3008990 T455281 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 3008992 T455343 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@79598ef name:ZooKeeperConnection Watcher:127.0.0.1:15150/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 3008992 T455281 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 3009014 T455281 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 3010017 T455281 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54345_x_%2Fh
   [junit4]   2> 3010019 T455281 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54345_x_%2Fh
   [junit4]   2> 3010023 T455303 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 3010023 T455325 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 3010023 T455312 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 3010023 T455343 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 3010035 T455344 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 3010035 T455344 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 3010036 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3010036 T455344 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 3010036 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3010037 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3010037 T455304 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3010038 T455304 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54345/x_/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54345_x_%2Fh",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 3010038 T455304 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
   [junit4]   2> 3010038 T455304 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 3010041 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3010050 T455303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 3010050 T455343 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 3010050 T455312 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 3010050 T455325 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 3011037 T455344 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 3011038 T455344 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 3011038 T455344 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 3011039 T455344 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 3011039 T455344 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 3011040 T455344 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-004/collection1/'
   [junit4]   2> 3011041 T455344 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 3011042 T455344 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 3011042 T455344 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 3011105 T455344 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 3011149 T455344 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 3011251 T455344 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 3011260 T455344 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 3011623 T455344 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 3011626 T455344 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 3011627 T455344 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 3011631 T455344 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 3011646 T455344 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 3011646 T455344 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 3011647 T455344 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-004/collection1/, dataDir=./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/jetty2/
   [junit4]   2> 3011647 T455344 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@61871708
   [junit4]   2> 3011648 T455344 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/jetty2
   [junit4]   2> 3011648 T455344 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/jetty2/index/
   [junit4]   2> 3011648 T455344 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 3011651 T455344 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.RecoveryZkTest-EFDCB4F4D6D1BBF-001/tempDir-001/jetty2/index
   [junit4]   2> 3011651 T455344 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=5.2294921875, floorSegmentMB=0.6162109375, forceMergeDeletesPctAllowed=26.593516805814566, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 3011652 T455344 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5a92c229 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@50992a80),segFN=segments_1,generation=1}
   [junit4]   2> 3011652 T455344 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 3011655 T455344 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 3011655 T455344 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 3011656 T455344 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 3011656 T455344 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 3011656 T455344 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 3011656 T455344 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 3011657 T455344 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 3011657 T455344 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 3011657 T455344 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 3011658 T455344 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 3011658 T455344 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 3011659 T455344 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 3011659 T455344 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 3011659 T455344 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 3011660 T455344 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 3011660 T455344 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 3011679 T455344 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 3011683 T455344 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 3011683 T455344 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 3011684 T455344 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=10.1904296875, floorSegmentMB=1.755859375, forceMergeDeletesPctAllowed=26.26056659028575, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.767914023905204
   [junit4]   2> 3011685 T455344 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5a92c229 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@50992a80),segFN=segments_1,generation=1}
   [junit4]   2> 3011685 T455344 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 3011685 T455344 oass.SolrIndexSearcher.<init> Opening Searcher@14952904[collection1] main
   [junit4]   2> 3011686 T455344 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 3011686 T455344 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 3011687 T455344 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 3011687 T455344 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 3011687 T455344 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 3011687 T455344 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 3011688 T455344 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 3011688 T455344 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 3011688 T455344 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 3011689 T455344 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 3011691 T455345 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@14952904[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 3011693 T455344 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 3011694 T455348 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54345/x_/h collection:collection1 shard:shard1
   [junit4]   2> 3011694 T455281 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
   [junit4]   2> 3011694 T455281 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 3011696 T455281 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3011696 T455348 oasc.ZkController.register We are http://127.0.0.1:54345/x_/h/collection1/ and leader is http://127.0.0.1:50417/x_/h/collection1/
   [junit4]   2> 3011697 T455348 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54345/x_/h
   [junit4]   2> 3011697 T455348 oasc.ZkController.checkRecovery Core needs to recover:collection1
   [junit4]   2> 3011701 T455348 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
   [junit4]   2> ASYNC  NEW_CORE C22750 name=collection1 org.apache.solr.core.SolrCore@23ab6d96 url=http://127.0.0.1:54345/x_/h/collection1 node=127.0.0.1:54345_x_%2Fh C22750_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:54345/x_/h, core=collection1, node_name=127.0.0.1:54345_x_%2Fh}
   [junit4]   2> 3011701 T455351 C22750 P54345 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
   [junit4]   2> 3011704 T455348 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3011704 T455351 C22750 P54345 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
   [junit4]   2> 3011704 T455351 C22750 P54345 oasc.RecoveryStrategy.doRecovery Publishing state of core collection1 as recovering, leader is http://127.0.0.1:50417/x_/h/collection1/ and I am http://127.0.0.1:54345/x_/h/collection1/
   [junit4]   2> 3011706 T455351 C22750 P54345 oasc.ZkController.publish publishing core=collection1 state=recovering collection=collection1
   [junit4]   2> 3011708 T455351 C22750 P54345 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 3011710 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C22751 name=collection1 org.apache.solr.core.SolrCore@730af28d url=http://127.0.0.1:50360/x_/h/collection1 node=127.0.0.1:50360_x_%2Fh C22751_STATE=coll:control_collection core:collection1 props:{state=active, base_url=http://127.0.0.1:50360/x_/h, core=collection1, node_name=127.0.0.1:50360_x_%2Fh, leader=true}
   [junit4]   2> 3011710 T455293 C22751 P50360 oasup.LogUpdateProcessor.finish [collection1] webapp=/x_/h path=/update params={version=2&wt=javabin&CONTROL=TRUE} {add=[2-0 (1472571095957635072)]} 0 6
   [junit4]   2> 3011710 T455351 C22750 P54345 oasc.RecoveryStrategy.sendPrepRecoveryCmd Sending prep recovery command to http://127.0.0.1:50417/x_/h; WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1%3A54345_x_%252Fh&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true
   [junit4]   2> 3011710 T455294 C22751 P50360 oasup.LogUpdateProcessor.finish [collection1] webapp=/x_/h path=/update params={version=2&wt=javabin&CONTROL=TRUE} {add=[1-0 (1472571095957635073)]} 0 6
   [junit4]   2> 3011711 T455304 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3011712 T455315 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 3011712 T455304 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54345/x_/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54345_x_%2Fh",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 3011715 T455315 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 3011720 T455315 oasha.CoreAdminHandler.handleWaitForStateAction Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
   [junit4]   2> 3011721 T455315 oasha.CoreAdminHandler.handleWaitForStateAction In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:54345_x_%2Fh, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"state":"down","base_url":"http://127.0.0.1:54345/x_/h","core":"collection1","node_name":"127.0.0.1:54345_x_%2Fh"}
   [junit4]   2> ASYNC  NEW_CORE C22752 name=collection1 org.apache.solr.core.SolrCore@7d56b881 url=http://127.0.0.1:50417/x_/h/collection1 node=127.0.0.1:50417_x_%2Fh C22752_STATE=coll:collection1 core:collection1 props:{state=active, base_url=http://127.0.0.1:50417/x_/h, core=collection1, node_name=127.0.0.1:50417_x_%2Fh, leader=true}
   [junit4]   2> 3011721 T455316 C22752 P50417 oasup.LogUpdateProcessor.finish [collection1] webapp=/x_/h path=/update params={version=2&wt=javabin} {add=[2-0 (1472571095969169408)]} 0 9
   [junit4]   2> 3011721 T455303 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 3011721 T455317 C22752 P50417 oasup.LogUpdateProcessor.finish [collection1] webapp=/x_/h path=/update params={version=2&wt=javabin} {add=[1-0 (1472571095970217984)]} 0 6
   [junit4]   2> 3011729 T455303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 3011729 T455343 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 3011729 T4553

[...truncated too long message...]

ngsFormat(name=Asserting), 46parent_f2_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 5grand3_s=PostingsFormat(name=FSTOrdPulsing41), 52parent_f1_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 39parent_f2_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 14parent_f2_s=PostingsFormat(name=FSTOrdPulsing41), 3grand0_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 12parent_f1_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 7grand4_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 38parent_f1_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 6parent_f2_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 49parent_f2_s=PostingsFormat(name=Asserting), 19parent_f2_s=MockVariableIntBlock(baseBlockSize=124), 6parent_f1_s=PostingsFormat(name=Asserting), 28parent_f2_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 2child0_s=PostingsFormat(name=FSTOrdPulsing41), 31parent_f1_s=PostingsFormat(name=FSTOrdPulsing41), 1parent_f1_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 17parent_f1_s=PostingsFormat(name=Asserting), 17parent_f2_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 48parent_f2_s=MockVariableIntBlock(baseBlockSize=124), 5grand2_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 1grand0_s=MockVariableIntBlock(baseBlockSize=124), 2grand1_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 5grand4_s=MockVariableIntBlock(baseBlockSize=124), 51parent_f1_s=PostingsFormat(name=Asserting), 31parent_f2_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 28parent_f1_s=PostingsFormat(name=Asserting), 7grand6_s=MockVariableIntBlock(baseBlockSize=124), 7grand1_s=PostingsFormat(name=FSTOrdPulsing41), 10parent_f1_s=MockVariableIntBlock(baseBlockSize=124), 7parent_f1_s=MockVariableIntBlock(baseBlockSize=124), 22parent_f2_s=MockVariableIntBlock(baseBlockSize=124), 2parent_f1_s=PostingsFormat(name=Asserting), 46parent_f1_s=PostingsFormat(name=FSTOrdPulsing41), 43parent_f1_s=MockVariableIntBlock(baseBlockSize=124), 25parent_f2_s=PostingsFormat(name=FSTOrdPulsing41), 16parent_f2_s=PostingsFormat(name=FSTOrdPulsing41), text_t=PostingsFormat(name=Asserting), 29parent_f2_s=PostingsFormat(name=Asserting), 6grand1_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 36parent_f2_s=PostingsFormat(name=FSTOrdPulsing41), 32parent_f2_s=PostingsFormat(name=FSTOrdPulsing41), 0parent_f1_s=PostingsFormat(name=FSTOrdPulsing41), 12parent_f2_s=PostingsFormat(name=Asserting), _version_=PostingsFormat(name=Asserting), 4parent_f1_s=PostingsFormat(name=FSTOrdPulsing41), 2child1_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 6grand4_s=PostingsFormat(name=FSTOrdPulsing41), 13parent_f1_s=PostingsFormat(name=FSTOrdPulsing41), 14parent_f1_s=MockVariableIntBlock(baseBlockSize=124), 45parent_f1_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 29parent_f1_s=MockVariableIntBlock(baseBlockSize=124), 27parent_f2_s=PostingsFormat(name=Asserting), 5parent_f1_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 4child0_s=PostingsFormat(name=FSTOrdPulsing41), 40parent_f2_s=MockVariableIntBlock(baseBlockSize=124), 8parent_f2_s=MockVariableIntBlock(baseBlockSize=124), 54parent_f2_s=PostingsFormat(name=FSTOrdPulsing41), 7grand0_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 22parent_f1_s=PostingsFormat(name=Asserting), 50parent_f1_s=MockVariableIntBlock(baseBlockSize=124), 23parent_f2_s=PostingsFormat(name=Asserting), 3grand1_s=PostingsFormat(name=FSTOrdPulsing41), 54parent_f1_s=MockVariableIntBlock(baseBlockSize=124), 41parent_f1_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 23parent_f1_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 35parent_f1_s=PostingsFormat(name=FSTOrdPulsing41), 24parent_f2_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 4grand1_s=MockVariableIntBlock(baseBlockSize=124), 33parent_f2_s=MockVariableIntBlock(baseBlockSize=124), id=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 18parent_f1_s=MockVariableIntBlock(baseBlockSize=124), 4child1_s=MockVariableIntBlock(baseBlockSize=124), 30parent_f2_s=PostingsFormat(name=Asserting), 33parent_f1_s=PostingsFormat(name=Asserting), 43parent_f2_s=PostingsFormat(name=FSTOrdPulsing41), 37parent_f2_s=MockVariableIntBlock(baseBlockSize=124), 7grand2_s=MockVariableIntBlock(baseBlockSize=124), 3child1_s=PostingsFormat(name=FSTOrdPulsing41), 55parent_f1_s=PostingsFormat(name=Asserting), 3parent_f1_s=MockVariableIntBlock(baseBlockSize=124), 53parent_f2_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 7parent_f2_s=PostingsFormat(name=Asserting), 44parent_f2_s=MockVariableIntBlock(baseBlockSize=124), 37parent_f1_s=PostingsFormat(name=Asserting), rnd_b=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 56parent_f2_s=PostingsFormat(name=Asserting), 25parent_f1_s=MockVariableIntBlock(baseBlockSize=124), 4child3_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 44parent_f1_s=PostingsFormat(name=Asserting), 26parent_f1_s=PostingsFormat(name=Asserting), 36parent_f1_s=MockVariableIntBlock(baseBlockSize=124), 55parent_f2_s=MockVariableIntBlock(baseBlockSize=124), 9parent_f1_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 4parent_f2_s=MockVariableIntBlock(baseBlockSize=124), 35parent_f2_s=Pulsing41(freqCutoff=17 minBlockSize=56 maxBlockSize=208), 21parent_f2_s=PostingsFormat(name=FSTOrdPulsing41), 3parent_f2_s=PostingsFormat(name=Asserting)}, docValues:{}, sim=DefaultSimilarity, locale=es_DO, timezone=America/Indiana/Knox
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=1,free=238880984,total=477102080
   [junit4]   2> NOTE: All tests run in this JVM: [AnalysisAfterCoreReloadTest, StatsComponentTest, TestSystemIdResolver, TestCoreContainer, TestAddFieldRealTimeGet, LeaderElectionIntegrationTest, TestIndexSearcher, DistributedQueryElevationComponentTest, TestCollationFieldDocValues, CSVRequestHandlerTest, HdfsUnloadDistributedZkTest, TestCursorMarkWithoutUniqueKey, TestElisionMultitermQuery, TestMiniSolrCloudCluster, AliasIntegrationTest, InfoHandlerTest, DirectSolrConnectionTest, FileUtilsTest, DistributedSuggestComponentTest, TestClassNameShortening, BJQParserTest, TestFastLRUCache, FieldAnalysisRequestHandlerTest, TestCollationKeyRangeQueries, AddSchemaFieldsUpdateProcessorFactoryTest, TestCSVLoader, TestDocumentBuilder, TestMultiCoreConfBootstrap, CollectionsAPIAsyncDistributedZkTest, HighlighterMaxOffsetTest, DocExpirationUpdateProcessorFactoryTest, TestGroupingSearch, HdfsDirectoryTest, TestFunctionQuery, DebugComponentTest, TestConfigSets, TestNonDefinedSimilarityFactory, TestXIncludeConfig, TestIBSimilarityFactory, TestLRUCache, TermVectorComponentDistributedTest, RequestHandlersTest, ShardRoutingCustomTest, TestManagedResourceStorage, HardAutoCommitTest, CurrencyFieldXmlFileTest, AssignTest, TestRecovery, SolrCoreTest, SearchHandlerTest, BasicDistributedZk2Test, SolrRequestParserTest, TestPHPSerializedResponseWriter, TestBinaryField, TestCSVResponseWriter, TestPerFieldSimilarity, AlternateDirectoryTest, TestImplicitCoreProperties, TestManagedStopFilterFactory, URLClassifyProcessorTest, TestValueSourceCache, SolrInfoMBeanTest, TestMergePolicyConfig, SolrPluginUtilsTest, TestCollationField, MultiThreadedOCPTest, BinaryUpdateRequestHandlerTest, TestFieldTypeCollectionResource, QueryParsingTest, PreAnalyzedFieldTest, CursorPagingTest, TestQueryTypes, CoreAdminRequestStatusTest, TestRTGBase, TestSchemaSimilarityResource, ShardSplitTest, TestDistribDocBasedVersion, TestSolrXmlPersistor, TermVectorComponentTest, TestCollapseQParserPlugin, LegacyHTMLStripCharFilterTest, TestDynamicFieldCollectionResource, TestWriterPerf, NumericFieldsTest, DirectUpdateHandlerOptimizeTest, DistributedTermsComponentTest, TestLeaderElectionZkExpiry, TestFreeTextSuggestions, DefaultValueUpdateProcessorTest, TestNonNRTOpen, TestUpdate, TestRealTimeGet, DeleteReplicaTest, SuggesterFSTTest, TestManagedSchemaFieldResource, CopyFieldTest, PolyFieldTest, ChaosMonkeySafeLeaderTest, CollectionsAPIDistributedZkTest, OpenCloseCoreStressTest, ShardRoutingTest, BasicZkTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest]
   [junit4] Completed on J1 in 111.81s, 1 test, 1 error <<< FAILURES!

[...truncated 643 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:474: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/common-build.xml:496: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1299: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:923: There were test failures: 404 suites, 1654 tests, 1 error, 1 failure, 50 ignored (8 assumptions)

Total time: 238 minutes 37 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-4.x #567
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 389990826 bytes
Compression is 0.0%
Took 1 min 6 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 573 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/573/

5 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
shard2 is not consistent.  Got 161 from http://127.0.0.1:43437/uzol/x/collection1lastClient and got 149 from http://127.0.0.1:48594/uzol/x/collection1

Stack Trace:
java.lang.AssertionError: shard2 is not consistent.  Got 161 from http://127.0.0.1:43437/uzol/x/collection1lastClient and got 149 from http://127.0.0.1:48594/uzol/x/collection1
	at __randomizedtesting.SeedInfo.seed([1851694D6BB338A8:99B7E7551CEC5894]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1139)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1118)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:150)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.cloud.CollectionsAPIAsyncDistributedZkTest.testDistribSearch

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([1851694D6BB338A8]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIAsyncDistributedZkTest

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([1851694D6BB338A8]:0)


REGRESSION:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
No registered leader was found after waiting for 60000ms , collection: c8n_1x3_lf slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 60000ms , collection: c8n_1x3_lf slice: shard1
	at __randomizedtesting.SeedInfo.seed([1851694D6BB338A8:99B7E7551CEC5894]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:545)
	at org.apache.solr.cloud.HttpPartitionTest.testRf3WithLeaderFailover(HttpPartitionTest.java:350)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:149)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1404758663793,generation=2,filelist=[_al.cfe, _al.cfs, _al.si, _l6.cfe, _l6.cfs, _l6.si, _lq.cfe, _lq.cfs, _lq.si, _ma.cfe, _ma.cfs, _ma.si, _mu.cfe, _mu.cfs, _mu.si, _ne.cfe, _ne.cfs, _ne.si, _ny.cfe, _ny.cfs, _ny.si, _oi.cfe, _oi.cfs, _oi.si, _p2.cfe, _p2.cfs, _p2.si, _pm.cfe, _pm.cfs, _pm.si, _q6.cfe, _q6.cfs, _q6.si, _qq.cfe, _qq.cfs, _qq.si, _ra.cfe, _ra.cfs, _ra.si, _ru.cfe, _ru.cfs, _ru.si, _se.cfe, _se.cfs, _se.si, _sy.cfe, _sy.cfs, _sy.si, _th.fdt, _th.fdx, _th.fnm, _th.nvd, _th.nvm, _th.si, _th_Lucene41_0.doc, _th_Lucene41_0.tim, _th_Lucene41_0.tip, _ti.cfe, _ti.cfs, _ti.si, _tj.fdt, _tj.fdx, _tj.fnm, _tj.nvd, _tj.nvm, _tj.si, _tj_Lucene41_0.doc, _tj_Lucene41_0.tim, _tj_Lucene41_0.tip, _tk.fdt, _tk.fdx, _tk.fnm, _tk.nvd, _tk.nvm, _tk.si, _tk_Lucene41_0.doc, _tk_Lucene41_0.tim, _tk_Lucene41_0.tip, _tl.fdt, _tl.fdx, _tl.fnm, _tl.nvd, _tl.nvm, _tl.si, _tl_Lucene41_0.doc, _tl_Lucene41_0.tim, _tl_Lucene41_0.tip, _tm.fdt, _tm.fdx, _tm.fnm, _tm.nvd, _tm.nvm, _tm.si, _tm_Lucene41_0.doc, _tm_Lucene41_0.tim, _tm_Lucene41_0.tip, _tn.fdt, _tn.fdx, _tn.fnm, _tn.nvd, _tn.nvm, _tn.si, _tn_Lucene41_0.doc, _tn_Lucene41_0.tim, _tn_Lucene41_0.tip, _to.fdt, _to.fdx, _to.fnm, _to.nvd, _to.nvm, _to.si, _to_Lucene41_0.doc, _to_Lucene41_0.tim, _to_Lucene41_0.tip, _tp.fdt, _tp.fdx, _tp.fnm, _tp.nvd, _tp.nvm, _tp.si, _tp_Lucene41_0.doc, _tp_Lucene41_0.tim, _tp_Lucene41_0.tip, _tq.fdt, _tq.fdx, _tq.fnm, _tq.nvd, _tq.nvm, _tq.si, _tq_Lucene41_0.doc, _tq_Lucene41_0.tim, _tq_Lucene41_0.tip, _tr.fdt, _tr.fdx, _tr.fnm, _tr.nvd, _tr.nvm, _tr.si, _tr_Lucene41_0.doc, _tr_Lucene41_0.tim, _tr_Lucene41_0.tip, _ts.fdt, _ts.fdx, _ts.fnm, _ts.nvd, _ts.nvm, _ts.si, _ts_Lucene41_0.doc, _ts_Lucene41_0.tim, _ts_Lucene41_0.tip, _tt.fdt, _tt.fdx, _tt.fnm, _tt.nvd, _tt.nvm, _tt.si, _tt_Lucene41_0.doc, _tt_Lucene41_0.tim, _tt_Lucene41_0.tip, _tu.fdt, _tu.fdx, _tu.fnm, _tu.nvd, _tu.nvm, _tu.si, _tu_Lucene41_0.doc, _tu_Lucene41_0.tim, _tu_Lucene41_0.tip, _tv.fdt, _tv.fdx, _tv.fnm, _tv.nvd, _tv.nvm, _tv.si, _tv_Lucene41_0.doc, _tv_Lucene41_0.tim, _tv_Lucene41_0.tip, _tw.fdt, _tw.fdx, _tw.fnm, _tw.nvd, _tw.nvm, _tw.si, _tw_Lucene41_0.doc, _tw_Lucene41_0.tim, _tw_Lucene41_0.tip, _tx.fdt, _tx.fdx, _tx.fnm, _tx.nvd, _tx.nvm, _tx.si, _tx_Lucene41_0.doc, _tx_Lucene41_0.tim, _tx_Lucene41_0.tip, _ty.fdt, _ty.fdx, _ty.fnm, _ty.nvd, _ty.nvm, _ty.si, _ty_Lucene41_0.doc, _ty_Lucene41_0.tim, _ty_Lucene41_0.tip, _tz.fdt, _tz.fdx, _tz.fnm, _tz.nvd, _tz.nvm, _tz.si, _tz_Lucene41_0.doc, _tz_Lucene41_0.tim, _tz_Lucene41_0.tip, _u0.fdt, _u0.fdx, _u0.fnm, _u0.nvd, _u0.nvm, _u0.si, _u0_Lucene41_0.doc, _u0_Lucene41_0.tim, _u0_Lucene41_0.tip, segments_2]}]> but was:<[{indexVersion=1404758663793,generation=2,filelist=[_al.cfe, _al.cfs, _al.si, _l6.cfe, _l6.cfs, _l6.si, _lq.cfe, _lq.cfs, _lq.si, _ma.cfe, _ma.cfs, _ma.si, _mu.cfe, _mu.cfs, _mu.si, _ne.cfe, _ne.cfs, _ne.si, _ny.cfe, _ny.cfs, _ny.si, _oi.cfe, _oi.cfs, _oi.si, _p2.cfe, _p2.cfs, _p2.si, _pm.cfe, _pm.cfs, _pm.si, _q6.cfe, _q6.cfs, _q6.si, _qq.cfe, _qq.cfs, _qq.si, _ra.cfe, _ra.cfs, _ra.si, _ru.cfe, _ru.cfs, _ru.si, _se.cfe, _se.cfs, _se.si, _sy.cfe, _sy.cfs, _sy.si, _th.fdt, _th.fdx, _th.fnm, _th.nvd, _th.nvm, _th.si, _th_Lucene41_0.doc, _th_Lucene41_0.tim, _th_Lucene41_0.tip, _ti.cfe, _ti.cfs, _ti.si, _tj.fdt, _tj.fdx, _tj.fnm, _tj.nvd, _tj.nvm, _tj.si, _tj_Lucene41_0.doc, _tj_Lucene41_0.tim, _tj_Lucene41_0.tip, _tk.fdt, _tk.fdx, _tk.fnm, _tk.nvd, _tk.nvm, _tk.si, _tk_Lucene41_0.doc, _tk_Lucene41_0.tim, _tk_Lucene41_0.tip, _tl.fdt, _tl.fdx, _tl.fnm, _tl.nvd, _tl.nvm, _tl.si, _tl_Lucene41_0.doc, _tl_Lucene41_0.tim, _tl_Lucene41_0.tip, _tm.fdt, _tm.fdx, _tm.fnm, _tm.nvd, _tm.nvm, _tm.si, _tm_Lucene41_0.doc, _tm_Lucene41_0.tim, _tm_Lucene41_0.tip, _tn.fdt, _tn.fdx, _tn.fnm, _tn.nvd, _tn.nvm, _tn.si, _tn_Lucene41_0.doc, _tn_Lucene41_0.tim, _tn_Lucene41_0.tip, _to.fdt, _to.fdx, _to.fnm, _to.nvd, _to.nvm, _to.si, _to_Lucene41_0.doc, _to_Lucene41_0.tim, _to_Lucene41_0.tip, _tp.fdt, _tp.fdx, _tp.fnm, _tp.nvd, _tp.nvm, _tp.si, _tp_Lucene41_0.doc, _tp_Lucene41_0.tim, _tp_Lucene41_0.tip, _tq.fdt, _tq.fdx, _tq.fnm, _tq.nvd, _tq.nvm, _tq.si, _tq_Lucene41_0.doc, _tq_Lucene41_0.tim, _tq_Lucene41_0.tip, _tr.fdt, _tr.fdx, _tr.fnm, _tr.nvd, _tr.nvm, _tr.si, _tr_Lucene41_0.doc, _tr_Lucene41_0.tim, _tr_Lucene41_0.tip, _ts.fdt, _ts.fdx, _ts.fnm, _ts.nvd, _ts.nvm, _ts.si, _ts_Lucene41_0.doc, _ts_Lucene41_0.tim, _ts_Lucene41_0.tip, _tt.fdt, _tt.fdx, _tt.fnm, _tt.nvd, _tt.nvm, _tt.si, _tt_Lucene41_0.doc, _tt_Lucene41_0.tim, _tt_Lucene41_0.tip, _tu.fdt, _tu.fdx, _tu.fnm, _tu.nvd, _tu.nvm, _tu.si, _tu_Lucene41_0.doc, _tu_Lucene41_0.tim, _tu_Lucene41_0.tip, _tv.fdt, _tv.fdx, _tv.fnm, _tv.nvd, _tv.nvm, _tv.si, _tv_Lucene41_0.doc, _tv_Lucene41_0.tim, _tv_Lucene41_0.tip, _tw.fdt, _tw.fdx, _tw.fnm, _tw.nvd, _tw.nvm, _tw.si, _tw_Lucene41_0.doc, _tw_Lucene41_0.tim, _tw_Lucene41_0.tip, _tx.fdt, _tx.fdx, _tx.fnm, _tx.nvd, _tx.nvm, _tx.si, _tx_Lucene41_0.doc, _tx_Lucene41_0.tim, _tx_Lucene41_0.tip, _ty.fdt, _ty.fdx, _ty.fnm, _ty.nvd, _ty.nvm, _ty.si, _ty_Lucene41_0.doc, _ty_Lucene41_0.tim, _ty_Lucene41_0.tip, _tz.fdt, _tz.fdx, _tz.fnm, _tz.nvd, _tz.nvm, _tz.si, _tz_Lucene41_0.doc, _tz_Lucene41_0.tim, _tz_Lucene41_0.tip, _u0.fdt, _u0.fdx, _u0.fnm, _u0.nvd, _u0.nvm, _u0.si, _u0_Lucene41_0.doc, _u0_Lucene41_0.tim, _u0_Lucene41_0.tip, segments_2]}, {indexVersion=1404758663793,generation=3,filelist=[_al.cfe, _al.cfs, _al.si, _l6.cfe, _l6.cfs, _l6.si, _lq.cfe, _lq.cfs, _lq.si, _ma.cfe, _ma.cfs, _ma.si, _mu.cfe, _mu.cfs, _mu.si, _ne.cfe, _ne.cfs, _ne.si, _ny.cfe, _ny.cfs, _ny.si, _oi.cfe, _oi.cfs, _oi.si, _p2.cfe, _p2.cfs, _p2.si, _pm.cfe, _pm.cfs, _pm.si, _q6.cfe, _q6.cfs, _q6.si, _qq.cfe, _qq.cfs, _qq.si, _ra.cfe, _ra.cfs, _ra.si, _ru.cfe, _ru.cfs, _ru.si, _se.cfe, _se.cfs, _se.si, _sy.cfe, _sy.cfs, _sy.si, _ti.cfe, _ti.cfs, _ti.si, _u1.cfe, _u1.cfs, _u1.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1404758663793,generation=2,filelist=[_al.cfe, _al.cfs, _al.si, _l6.cfe, _l6.cfs, _l6.si, _lq.cfe, _lq.cfs, _lq.si, _ma.cfe, _ma.cfs, _ma.si, _mu.cfe, _mu.cfs, _mu.si, _ne.cfe, _ne.cfs, _ne.si, _ny.cfe, _ny.cfs, _ny.si, _oi.cfe, _oi.cfs, _oi.si, _p2.cfe, _p2.cfs, _p2.si, _pm.cfe, _pm.cfs, _pm.si, _q6.cfe, _q6.cfs, _q6.si, _qq.cfe, _qq.cfs, _qq.si, _ra.cfe, _ra.cfs, _ra.si, _ru.cfe, _ru.cfs, _ru.si, _se.cfe, _se.cfs, _se.si, _sy.cfe, _sy.cfs, _sy.si, _th.fdt, _th.fdx, _th.fnm, _th.nvd, _th.nvm, _th.si, _th_Lucene41_0.doc, _th_Lucene41_0.tim, _th_Lucene41_0.tip, _ti.cfe, _ti.cfs, _ti.si, _tj.fdt, _tj.fdx, _tj.fnm, _tj.nvd, _tj.nvm, _tj.si, _tj_Lucene41_0.doc, _tj_Lucene41_0.tim, _tj_Lucene41_0.tip, _tk.fdt, _tk.fdx, _tk.fnm, _tk.nvd, _tk.nvm, _tk.si, _tk_Lucene41_0.doc, _tk_Lucene41_0.tim, _tk_Lucene41_0.tip, _tl.fdt, _tl.fdx, _tl.fnm, _tl.nvd, _tl.nvm, _tl.si, _tl_Lucene41_0.doc, _tl_Lucene41_0.tim, _tl_Lucene41_0.tip, _tm.fdt, _tm.fdx, _tm.fnm, _tm.nvd, _tm.nvm, _tm.si, _tm_Lucene41_0.doc, _tm_Lucene41_0.tim, _tm_Lucene41_0.tip, _tn.fdt, _tn.fdx, _tn.fnm, _tn.nvd, _tn.nvm, _tn.si, _tn_Lucene41_0.doc, _tn_Lucene41_0.tim, _tn_Lucene41_0.tip, _to.fdt, _to.fdx, _to.fnm, _to.nvd, _to.nvm, _to.si, _to_Lucene41_0.doc, _to_Lucene41_0.tim, _to_Lucene41_0.tip, _tp.fdt, _tp.fdx, _tp.fnm, _tp.nvd, _tp.nvm, _tp.si, _tp_Lucene41_0.doc, _tp_Lucene41_0.tim, _tp_Lucene41_0.tip, _tq.fdt, _tq.fdx, _tq.fnm, _tq.nvd, _tq.nvm, _tq.si, _tq_Lucene41_0.doc, _tq_Lucene41_0.tim, _tq_Lucene41_0.tip, _tr.fdt, _tr.fdx, _tr.fnm, _tr.nvd, _tr.nvm, _tr.si, _tr_Lucene41_0.doc, _tr_Lucene41_0.tim, _tr_Lucene41_0.tip, _ts.fdt, _ts.fdx, _ts.fnm, _ts.nvd, _ts.nvm, _ts.si, _ts_Lucene41_0.doc, _ts_Lucene41_0.tim, _ts_Lucene41_0.tip, _tt.fdt, _tt.fdx, _tt.fnm, _tt.nvd, _tt.nvm, _tt.si, _tt_Lucene41_0.doc, _tt_Lucene41_0.tim, _tt_Lucene41_0.tip, _tu.fdt, _tu.fdx, _tu.fnm, _tu.nvd, _tu.nvm, _tu.si, _tu_Lucene41_0.doc, _tu_Lucene41_0.tim, _tu_Lucene41_0.tip, _tv.fdt, _tv.fdx, _tv.fnm, _tv.nvd, _tv.nvm, _tv.si, _tv_Lucene41_0.doc, _tv_Lucene41_0.tim, _tv_Lucene41_0.tip, _tw.fdt, _tw.fdx, _tw.fnm, _tw.nvd, _tw.nvm, _tw.si, _tw_Lucene41_0.doc, _tw_Lucene41_0.tim, _tw_Lucene41_0.tip, _tx.fdt, _tx.fdx, _tx.fnm, _tx.nvd, _tx.nvm, _tx.si, _tx_Lucene41_0.doc, _tx_Lucene41_0.tim, _tx_Lucene41_0.tip, _ty.fdt, _ty.fdx, _ty.fnm, _ty.nvd, _ty.nvm, _ty.si, _ty_Lucene41_0.doc, _ty_Lucene41_0.tim, _ty_Lucene41_0.tip, _tz.fdt, _tz.fdx, _tz.fnm, _tz.nvd, _tz.nvm, _tz.si, _tz_Lucene41_0.doc, _tz_Lucene41_0.tim, _tz_Lucene41_0.tip, _u0.fdt, _u0.fdx, _u0.fnm, _u0.nvd, _u0.nvm, _u0.si, _u0_Lucene41_0.doc, _u0_Lucene41_0.tim, _u0_Lucene41_0.tip, segments_2]}]> but was:<[{indexVersion=1404758663793,generation=2,filelist=[_al.cfe, _al.cfs, _al.si, _l6.cfe, _l6.cfs, _l6.si, _lq.cfe, _lq.cfs, _lq.si, _ma.cfe, _ma.cfs, _ma.si, _mu.cfe, _mu.cfs, _mu.si, _ne.cfe, _ne.cfs, _ne.si, _ny.cfe, _ny.cfs, _ny.si, _oi.cfe, _oi.cfs, _oi.si, _p2.cfe, _p2.cfs, _p2.si, _pm.cfe, _pm.cfs, _pm.si, _q6.cfe, _q6.cfs, _q6.si, _qq.cfe, _qq.cfs, _qq.si, _ra.cfe, _ra.cfs, _ra.si, _ru.cfe, _ru.cfs, _ru.si, _se.cfe, _se.cfs, _se.si, _sy.cfe, _sy.cfs, _sy.si, _th.fdt, _th.fdx, _th.fnm, _th.nvd, _th.nvm, _th.si, _th_Lucene41_0.doc, _th_Lucene41_0.tim, _th_Lucene41_0.tip, _ti.cfe, _ti.cfs, _ti.si, _tj.fdt, _tj.fdx, _tj.fnm, _tj.nvd, _tj.nvm, _tj.si, _tj_Lucene41_0.doc, _tj_Lucene41_0.tim, _tj_Lucene41_0.tip, _tk.fdt, _tk.fdx, _tk.fnm, _tk.nvd, _tk.nvm, _tk.si, _tk_Lucene41_0.doc, _tk_Lucene41_0.tim, _tk_Lucene41_0.tip, _tl.fdt, _tl.fdx, _tl.fnm, _tl.nvd, _tl.nvm, _tl.si, _tl_Lucene41_0.doc, _tl_Lucene41_0.tim, _tl_Lucene41_0.tip, _tm.fdt, _tm.fdx, _tm.fnm, _tm.nvd, _tm.nvm, _tm.si, _tm_Lucene41_0.doc, _tm_Lucene41_0.tim, _tm_Lucene41_0.tip, _tn.fdt, _tn.fdx, _tn.fnm, _tn.nvd, _tn.nvm, _tn.si, _tn_Lucene41_0.doc, _tn_Lucene41_0.tim, _tn_Lucene41_0.tip, _to.fdt, _to.fdx, _to.fnm, _to.nvd, _to.nvm, _to.si, _to_Lucene41_0.doc, _to_Lucene41_0.tim, _to_Lucene41_0.tip, _tp.fdt, _tp.fdx, _tp.fnm, _tp.nvd, _tp.nvm, _tp.si, _tp_Lucene41_0.doc, _tp_Lucene41_0.tim, _tp_Lucene41_0.tip, _tq.fdt, _tq.fdx, _tq.fnm, _tq.nvd, _tq.nvm, _tq.si, _tq_Lucene41_0.doc, _tq_Lucene41_0.tim, _tq_Lucene41_0.tip, _tr.fdt, _tr.fdx, _tr.fnm, _tr.nvd, _tr.nvm, _tr.si, _tr_Lucene41_0.doc, _tr_Lucene41_0.tim, _tr_Lucene41_0.tip, _ts.fdt, _ts.fdx, _ts.fnm, _ts.nvd, _ts.nvm, _ts.si, _ts_Lucene41_0.doc, _ts_Lucene41_0.tim, _ts_Lucene41_0.tip, _tt.fdt, _tt.fdx, _tt.fnm, _tt.nvd, _tt.nvm, _tt.si, _tt_Lucene41_0.doc, _tt_Lucene41_0.tim, _tt_Lucene41_0.tip, _tu.fdt, _tu.fdx, _tu.fnm, _tu.nvd, _tu.nvm, _tu.si, _tu_Lucene41_0.doc, _tu_Lucene41_0.tim, _tu_Lucene41_0.tip, _tv.fdt, _tv.fdx, _tv.fnm, _tv.nvd, _tv.nvm, _tv.si, _tv_Lucene41_0.doc, _tv_Lucene41_0.tim, _tv_Lucene41_0.tip, _tw.fdt, _tw.fdx, _tw.fnm, _tw.nvd, _tw.nvm, _tw.si, _tw_Lucene41_0.doc, _tw_Lucene41_0.tim, _tw_Lucene41_0.tip, _tx.fdt, _tx.fdx, _tx.fnm, _tx.nvd, _tx.nvm, _tx.si, _tx_Lucene41_0.doc, _tx_Lucene41_0.tim, _tx_Lucene41_0.tip, _ty.fdt, _ty.fdx, _ty.fnm, _ty.nvd, _ty.nvm, _ty.si, _ty_Lucene41_0.doc, _ty_Lucene41_0.tim, _ty_Lucene41_0.tip, _tz.fdt, _tz.fdx, _tz.fnm, _tz.nvd, _tz.nvm, _tz.si, _tz_Lucene41_0.doc, _tz_Lucene41_0.tim, _tz_Lucene41_0.tip, _u0.fdt, _u0.fdx, _u0.fnm, _u0.nvd, _u0.nvm, _u0.si, _u0_Lucene41_0.doc, _u0_Lucene41_0.tim, _u0_Lucene41_0.tip, segments_2]}, {indexVersion=1404758663793,generation=3,filelist=[_al.cfe, _al.cfs, _al.si, _l6.cfe, _l6.cfs, _l6.si, _lq.cfe, _lq.cfs, _lq.si, _ma.cfe, _ma.cfs, _ma.si, _mu.cfe, _mu.cfs, _mu.si, _ne.cfe, _ne.cfs, _ne.si, _ny.cfe, _ny.cfs, _ny.si, _oi.cfe, _oi.cfs, _oi.si, _p2.cfe, _p2.cfs, _p2.si, _pm.cfe, _pm.cfs, _pm.si, _q6.cfe, _q6.cfs, _q6.si, _qq.cfe, _qq.cfs, _qq.si, _ra.cfe, _ra.cfs, _ra.si, _ru.cfe, _ru.cfs, _ru.si, _se.cfe, _se.cfs, _se.si, _sy.cfe, _sy.cfs, _sy.si, _ti.cfe, _ti.cfs, _ti.si, _u1.cfe, _u1.cfs, _u1.si, segments_3]}]>
	at __randomizedtesting.SeedInfo.seed([1851694D6BB338A8:3D86727D1BFB36AB]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1191)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11602 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/init-core-data-001
   [junit4]   2> 727144 T2470 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 727145 T2470 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /uzol/x
   [junit4]   2> 727151 T2470 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 727152 T2470 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 727153 T2471 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 727254 T2470 oasc.ZkTestServer.run start zk server on port:43430
   [junit4]   2> 727255 T2470 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 727260 T2477 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@930c9d8 name:ZooKeeperConnection Watcher:127.0.0.1:43430 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 727260 T2470 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 727260 T2470 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 727278 T2470 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 727281 T2479 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ffaa80e name:ZooKeeperConnection Watcher:127.0.0.1:43430/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 727281 T2470 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 727281 T2470 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 727290 T2470 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 727293 T2470 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 727300 T2470 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 727304 T2470 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 727304 T2470 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 727315 T2470 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 727316 T2470 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 727319 T2470 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 727320 T2470 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 727323 T2470 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 727323 T2470 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 727326 T2470 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 727326 T2470 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 727329 T2470 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 727329 T2470 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 727337 T2470 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 727338 T2470 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 727340 T2470 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 727341 T2470 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 727344 T2470 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 727345 T2470 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 727348 T2470 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 727348 T2470 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 727356 T2470 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 727356 T2470 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 728455 T2470 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 728459 T2470 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:43433
   [junit4]   2> 728459 T2470 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 728460 T2470 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 728460 T2470 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-002
   [junit4]   2> 728460 T2470 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-002/'
   [junit4]   2> 728489 T2470 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-002/solr.xml
   [junit4]   2> 728539 T2470 oasc.CoreContainer.<init> New CoreContainer 257980829
   [junit4]   2> 728539 T2470 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-002/]
   [junit4]   2> 728540 T2470 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 728541 T2470 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 728541 T2470 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 728541 T2470 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 728541 T2470 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 728541 T2470 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 728542 T2470 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 728542 T2470 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 728542 T2470 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 728543 T2470 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 728543 T2470 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 728543 T2470 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 728543 T2470 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:43430/solr
   [junit4]   2> 728544 T2470 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 728544 T2470 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 728547 T2490 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6d03a4e name:ZooKeeperConnection Watcher:127.0.0.1:43430 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 728547 T2470 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 728550 T2470 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 728552 T2492 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@76d88013 name:ZooKeeperConnection Watcher:127.0.0.1:43430/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 728552 T2470 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 728554 T2470 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 728568 T2470 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 728572 T2470 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 728585 T2470 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 728589 T2470 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 728593 T2470 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 728602 T2470 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43433_uzol%2Fx
   [junit4]   2> 728603 T2470 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43433_uzol%2Fx
   [junit4]   2> 728607 T2470 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 728616 T2470 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 728630 T2470 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:43433_uzol%2Fx
   [junit4]   2> 728630 T2470 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 728632 T2470 oasc.Overseer.start Overseer (id=92062095175450627-127.0.0.1:43433_uzol%2Fx-n_0000000000) starting
   [junit4]   2> 728635 T2470 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 728644 T2494 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 728645 T2470 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 728647 T2470 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 728649 T2470 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 728651 T2493 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 728655 T2495 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 728655 T2495 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 728657 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 728657 T2495 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 728658 T2493 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 728659 T2493 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43433/uzol/x",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:43433_uzol%2Fx",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 728659 T2493 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 728659 T2493 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 728662 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 728663 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 729658 T2495 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 729659 T2495 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 729660 T2495 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 729660 T2495 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 729661 T2495 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 729661 T2495 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-002/collection1/'
   [junit4]   2> 729662 T2495 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 729663 T2495 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 729664 T2495 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 729723 T2495 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 729766 T2495 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 729767 T2495 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 729777 T2495 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 730131 T2495 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 730134 T2495 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 730136 T2495 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 730140 T2495 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 730162 T2495 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 730163 T2495 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 730163 T2495 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-002/collection1/, dataDir=./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-001/control/data/
   [junit4]   2> 730163 T2495 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b1e452
   [junit4]   2> 730165 T2495 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-001/control/data
   [junit4]   2> 730166 T2495 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-001/control/data/index/
   [junit4]   2> 730167 T2495 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 730168 T2495 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-001/control/data/index
   [junit4]   2> 730169 T2495 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@16205e1b
   [junit4]   2> 730174 T2495 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 730174 T2495 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 730178 T2495 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 730178 T2495 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 730179 T2495 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 730179 T2495 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 730179 T2495 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 730180 T2495 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 730180 T2495 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 730180 T2495 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 730181 T2495 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 730181 T2495 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 730182 T2495 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 730182 T2495 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 730182 T2495 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 730182 T2495 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 730183 T2495 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 730183 T2495 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 730202 T2495 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 730205 T2495 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 730205 T2495 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 730206 T2495 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=6.4072265625, floorSegmentMB=2.1943359375, forceMergeDeletesPctAllowed=27.562110970337493, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.11954163353620767
   [junit4]   2> 730208 T2495 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 730208 T2495 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 730209 T2495 oass.SolrIndexSearcher.<init> Opening Searcher@f895124[collection1] main
   [junit4]   2> 730209 T2495 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 730210 T2495 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 730210 T2495 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 730211 T2495 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 730211 T2495 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 730211 T2495 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 730212 T2495 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 730212 T2495 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 730212 T2495 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 730213 T2495 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 730215 T2496 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@f895124[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 730217 T2495 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 730218 T2499 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:43433/uzol/x collection:control_collection shard:shard1
   [junit4]   2> 730219 T2470 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
   [junit4]   2> 730219 T2470 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 730219 T2499 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 730220 T2470 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 730223 T2501 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@751e4323 name:ZooKeeperConnection Watcher:127.0.0.1:43430/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 730223 T2470 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 730234 T2470 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 730242 T2470 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 730244 T2499 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 730254 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 730255 T2499 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 730255 T2499 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C289 name=collection1 org.apache.solr.core.SolrCore@5d4a13d3 url=http://127.0.0.1:43433/uzol/x/collection1 node=127.0.0.1:43433_uzol%2Fx C289_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:43433/uzol/x, core=collection1, node_name=127.0.0.1:43433_uzol%2Fx}
   [junit4]   2> 730255 T2499 C289 P43433 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:43433/uzol/x/collection1/
   [junit4]   2> 730255 T2499 C289 P43433 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 730255 T2493 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 730256 T2499 C289 P43433 oasc.SyncStrategy.syncToMe http://127.0.0.1:43433/uzol/x/collection1/ has no replicas
   [junit4]   2> 730256 T2499 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:43433/uzol/x/collection1/ shard1
   [junit4]   2> 730256 T2499 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 730265 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 730273 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 730273 T2501 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 730290 T2493 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 730294 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 730397 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 730397 T2501 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 730443 T2499 oasc.ZkController.register We are http://127.0.0.1:43433/uzol/x/collection1/ and leader is http://127.0.0.1:43433/uzol/x/collection1/
   [junit4]   2> 730444 T2499 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:43433/uzol/x
   [junit4]   2> 730444 T2499 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 730444 T2499 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 730444 T2499 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 730447 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 730447 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 730447 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 730447 T2499 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 730448 T2493 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 730449 T2493 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43433/uzol/x",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:43433_uzol%2Fx",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 730458 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 730561 T2501 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 730561 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 731229 T2470 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 731230 T2470 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 731233 T2470 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:43437
   [junit4]   2> 731233 T2470 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 731233 T2470 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 731234 T2470 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-003
   [junit4]   2> 731234 T2470 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-003/'
   [junit4]   2> 731262 T2470 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-003/solr.xml
   [junit4]   2> 731319 T2470 oasc.CoreContainer.<init> New CoreContainer 1981158284
   [junit4]   2> 731320 T2470 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-003/]
   [junit4]   2> 731321 T2470 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 731321 T2470 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 731321 T2470 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 731321 T2470 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 731322 T2470 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 731322 T2470 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 731322 T2470 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 731322 T2470 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 731323 T2470 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 731323 T2470 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 731323 T2470 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 731324 T2470 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 731324 T2470 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:43430/solr
   [junit4]   2> 731324 T2470 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 731325 T2470 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 731328 T2512 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@235d88e5 name:ZooKeeperConnection Watcher:127.0.0.1:43430 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 731328 T2470 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 731348 T2470 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 731350 T2514 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@155b23af name:ZooKeeperConnection Watcher:127.0.0.1:43430/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 731350 T2470 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 731373 T2470 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 732377 T2470 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43437_uzol%2Fx
   [junit4]   2> 732379 T2470 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43437_uzol%2Fx
   [junit4]   2> 732382 T2514 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 732382 T2492 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 732382 T2501 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 732390 T2515 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 732391 T2515 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 732392 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 732392 T2515 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 732392 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 732393 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 732394 T2493 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 732395 T2493 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43437/uzol/x",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:43437_uzol%2Fx",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 732395 T2493 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
   [junit4]   2> 732395 T2493 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 732399 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 732408 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 732408 T2501 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 732408 T2514 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 733393 T2515 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 733394 T2515 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 733395 T2515 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 733395 T2515 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 733396 T2515 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 733396 T2515 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-003/collection1/'
   [junit4]   2> 733398 T2515 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 733398 T2515 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 733399 T2515 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 733459 T2515 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 733502 T2515 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 733504 T2515 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 733513 T2515 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 733867 T2515 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 733870 T2515 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 733871 T2515 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 733876 T2515 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 733897 T2515 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 733898 T2515 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 733898 T2515 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-003/collection1/, dataDir=./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-001/jetty1/
   [junit4]   2> 733898 T2515 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b1e452
   [junit4]   2> 733900 T2515 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-001/jetty1
   [junit4]   2> 733901 T2515 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-001/jetty1/index/
   [junit4]   2> 733902 T2515 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 733903 T2515 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-001/jetty1/index
   [junit4]   2> 733903 T2515 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@17a6282
   [junit4]   2> 733911 T2515 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 733911 T2515 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 733916 T2515 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 733917 T2515 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 733917 T2515 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 733917 T2515 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 733918 T2515 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 733918 T2515 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 733918 T2515 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 733919 T2515 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 733919 T2515 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 733919 T2515 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 733920 T2515 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 733920 T2515 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 733921 T2515 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 733921 T2515 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 733922 T2515 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 733922 T2515 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 733941 T2515 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 733944 T2515 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 733944 T2515 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 733945 T2515 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=6.4072265625, floorSegmentMB=2.1943359375, forceMergeDeletesPctAllowed=27.562110970337493, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.11954163353620767
   [junit4]   2> 733947 T2515 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 733947 T2515 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 733947 T2515 oass.SolrIndexSearcher.<init> Opening Searcher@1cfdae43[collection1] main
   [junit4]   2> 733948 T2515 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 733949 T2515 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 733949 T2515 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 733949 T2515 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 733950 T2515 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 733950 T2515 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 733951 T2515 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 733951 T2515 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 733951 T2515 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 733951 T2515 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 733955 T2516 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1cfdae43[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 733958 T2515 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 733958 T2519 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:43437/uzol/x collection:collection1 shard:shard2
   [junit4]   2> 733959 T2470 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
   [junit4]   2> 733959 T2470 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 733959 T2519 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 733976 T2519 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 733979 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 733979 T2519 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 733979 T2519 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C290 name=collection1 org.apache.solr.core.SolrCore@77434c98 url=http://127.0.0.1:43437/uzol/x/collection1 node=127.0.0.1:43437_uzol%2Fx C290_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:43437/uzol/x, core=collection1, node_name=127.0.0.1:43437_uzol%2Fx}
   [junit4]   2> 733979 T2519 C290 P43437 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:43437/uzol/x/collection1/
   [junit4]   2> 733980 T2519 C290 P43437 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 733980 T2493 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 733980 T2519 C290 P43437 oasc.SyncStrategy.syncToMe http://127.0.0.1:43437/uzol/x/collection1/ has no replicas
   [junit4]   2> 733980 T2519 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:43437/uzol/x/collection1/ shard2
   [junit4]   2> 733981 T2519 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 733983 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 733984 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 733984 T2501 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 733984 T2514 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 733989 T2493 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 733992 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 734096 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 734096 T2501 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 734096 T2514 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 734142 T2519 oasc.ZkController.register We are http://127.0.0.1:43437/uzol/x/collection1/ and leader is http://127.0.0.1:43437/uzol/x/collection1/
   [junit4]   2> 734143 T2519 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:43437/uzol/x
   [junit4]   2> 734143 T2519 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 734143 T2519 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 734143 T2519 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 734145 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 734145 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 734145 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 734145 T2519 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 734147 T2493 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 734147 T2493 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43437/uzol/x",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:43437_uzol%2Fx",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 734151 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 734253 T2514 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 734253 T2501 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 734253 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 735039 T2470 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 735040 T2470 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 735042 T2470 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:43440
   [junit4]   2> 735043 T2470 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 735043 T2470 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 735043 T2470 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-004
   [junit4]   2> 735044 T2470 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-004/'
   [junit4]   2> 735072 T2470 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-004/solr.xml
   [junit4]   2> 735122 T2470 oasc.CoreContainer.<init> New CoreContainer 719027088
   [junit4]   2> 735123 T2470 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-004/]
   [junit4]   2> 735124 T2470 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 735124 T2470 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 735124 T2470 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 735124 T2470 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 735124 T2470 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 735125 T2470 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 735125 T2470 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 735125 T2470 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 735125 T2470 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 735126 T2470 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 735126 T2470 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 735126 T2470 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 735127 T2470 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:43430/solr
   [junit4]   2> 735127 T2470 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 735128 T2470 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 735158 T2530 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f8d40eb name:ZooKeeperConnection Watcher:127.0.0.1:43430 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 735158 T2470 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 735161 T2470 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 735172 T2532 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7ab293cb name:ZooKeeperConnection Watcher:127.0.0.1:43430/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 735172 T2470 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 735189 T2470 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 736193 T2470 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43440_uzol%2Fx
   [junit4]   2> 736195 T2470 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43440_uzol%2Fx
   [junit4]   2> 736198 T2501 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 736198 T2492 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 736198 T2532 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 736198 T2514 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 736214 T2533 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 736214 T2533 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 736216 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 736216 T2533 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 736216 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 736216 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 736217 T2493 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 736218 T2493 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43440/uzol/x",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:43440_uzol%2Fx",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 736218 T2493 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 736219 T2493 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 736228 T2492 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 736229 T2532 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 736229 T2514 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 736229 T2501 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 736229 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 737216 T2533 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 737217 T2533 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 737218 T2533 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 737218 T2533 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 737219 T2533 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 737219 T2533 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-004/collection1/'
   [junit4]   2> 737221 T2533 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 737222 T2533 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 737223 T2533 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-1851694D6BB338A8-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 737283 T2533 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 737327 T2533 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 737328 T2533 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 737338 T2533 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 737717 T2533 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 737720 T2533 oass.IndexSchema.readSche

[...truncated too long message...]

pache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
   [junit4]   2> 	        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	   8) Thread[id=16776, name=searcherExecutor-2270-thread-1, state=WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
   [junit4]   2> 	        at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]   2> 	        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]   2> 	        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	   9) Thread[id=16700, name=TEST-CollectionsAPIAsyncDistributedZkTest.testDistribSearch-seed#[1851694D6BB338A8]-SendThread(127.0.0.1:41115), state=RUNNABLE, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
   [junit4]   2> 	        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
   [junit4]   2> 	        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
   [junit4]   2> 	        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
   [junit4]   2> 	        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
   [junit4]   2> 	        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
   [junit4]   2> 	        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
   [junit4]   2> 	        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=230), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=230)), sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=ar_EG, timezone=Asia/Hovd
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=11,free=268502728,total=520617984
   [junit4]   2> NOTE: All tests run in this JVM: [TestWordDelimiterFilterFactory, TestConfig, TestNumberUtils, PreAnalyzedUpdateProcessorTest, TestSearchPerf, ExternalFileFieldSortTest, TestSolrIndexConfig, HighlighterTest, TestReload, TestDocSet, PrimUtilsTest, PathHierarchyTokenizerFactoryTest, OverseerStatusTest, TestStressReorder, IndexSchemaTest, ChangedSchemaMergeTest, MultiThreadedOCPTest, TestFastOutputStream, TestAddFieldRealTimeGet, RegexBoostProcessorTest, LeaderElectionTest, SpellPossibilityIteratorTest, ReplicationFactorTest, UUIDFieldTest, TestFileDictionaryLookup, TestHashPartitioner, FastVectorHighlighterTest, TestBM25SimilarityFactory, TestFieldResource, BasicDistributedZkTest, SolrIndexSplitterTest, SuggesterTSTTest, TestDistributedSearch, NoCacheHeaderTest, PingRequestHandlerTest, SolrCmdDistributorTest, TestDistribDocBasedVersion, DocumentAnalysisRequestHandlerTest, TestCollectionAPI, DocumentBuilderTest, HdfsDirectoryTest, PrimitiveFieldTypeTest, TestUniqueKeyFieldResource, TestSolrXmlPersistor, HdfsBasicDistributedZkTest, DeleteReplicaTest, TestFuzzyAnalyzedSuggestions, JSONWriterTest, HighlighterConfigTest, TestCodecSupport, TestSolrCoreProperties, PluginInfoTest, HdfsBasicDistributedZk2Test, TestCursorMarkWithoutUniqueKey, DocValuesMissingTest, TestReplicationHandler, TestStressRecovery, JsonLoaderTest, SuggesterTest, DistribCursorPagingTest, PeerSyncTest, CoreAdminRequestStatusTest, TestFieldCollectionResource, ConnectionManagerTest, StatelessScriptUpdateProcessorFactoryTest, ZkCLITest, TestQuerySenderListener, TestSchemaNameResource, TestSort, OverseerRolesTest, TestRequestStatusCollectionAPI, DirectUpdateHandlerTest, CurrencyFieldOpenExchangeTest, BlockDirectoryTest, DateMathParserTest, SliceStateTest, MinimalSchemaTest, SolrXmlInZkTest, SpellCheckComponentTest, CollectionsAPIAsyncDistributedZkTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CollectionsAPIAsyncDistributedZkTest -Dtests.seed=1851694D6BB338A8 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/hudson/lucene-data/enwiki.random.lines.txt -Dtests.locale=ar_EG -Dtests.timezone=Asia/Hovd -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | CollectionsAPIAsyncDistributedZkTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([1851694D6BB338A8]:0)
   [junit4] Completed on J0 in 7223.50s, 1 test, 2 errors <<< FAILURES!

[...truncated 71 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:474: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/common-build.xml:496: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1299: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:923: There were test failures: 405 suites, 1656 tests, 1 suite-level error, 2 errors, 2 failures, 94 ignored (7 assumptions)

Total time: 313 minutes 43 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-4.x #567
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 627167765 bytes
Compression is 0.0%
Took 1 min 48 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 572 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/572/

1 tests failed.
REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1404642157034,generation=2,filelist=[_90.fdt, _90.fdx, _90.fnm, _90.nvd, _90.nvm, _90.si, _90_Memory_0.ram, _91.cfe, _91.cfs, _91.si, _92.cfe, _92.cfs, _92.si, _93.cfe, _93.cfs, _93.si, _94.cfe, _94.cfs, _94.si, _95.cfe, _95.cfs, _95.si, _96.cfe, _96.cfs, _96.si, _97.cfe, _97.cfs, _97.si, _98.cfe, _98.cfs, _98.si, _99.cfe, _99.cfs, _99.si, _9a.cfe, _9a.cfs, _9a.si, _9b.cfe, _9b.cfs, _9b.si, _9c.cfe, _9c.cfs, _9c.si, _9d.cfe, _9d.cfs, _9d.si, _9e.cfe, _9e.cfs, _9e.si, _9f.cfe, _9f.cfs, _9f.si, _9g.cfe, _9g.cfs, _9g.si, _9h.cfe, _9h.cfs, _9h.si, _9i.cfe, _9i.cfs, _9i.si, _9j.cfe, _9j.cfs, _9j.si, _9k.cfe, _9k.cfs, _9k.si, _9l.cfe, _9l.cfs, _9l.si, _9m.cfe, _9m.cfs, _9m.si, _9n.cfe, _9n.cfs, _9n.si, _9o.cfe, _9o.cfs, _9o.si, _9p.cfe, _9p.cfs, _9p.si, _9q.cfe, _9q.cfs, _9q.si, _9r.cfe, _9r.cfs, _9r.si, _9s.cfe, _9s.cfs, _9s.si, _9t.cfe, _9t.cfs, _9t.si, _9u.cfe, _9u.cfs, _9u.si, _9v.cfe, _9v.cfs, _9v.si, _9w.cfe, _9w.cfs, _9w.si, _9x.cfe, _9x.cfs, _9x.si, _9y.cfe, _9y.cfs, _9y.si, _9z.cfe, _9z.cfs, _9z.si, segments_2]}]> but was:<[{indexVersion=1404642157034,generation=3,filelist=[_a0.fdt, _a0.fdx, _a0.fnm, _a0.nvd, _a0.nvm, _a0.si, _a0_Memory_0.ram, segments_3]}, {indexVersion=1404642157034,generation=2,filelist=[_90.fdt, _90.fdx, _90.fnm, _90.nvd, _90.nvm, _90.si, _90_Memory_0.ram, _91.cfe, _91.cfs, _91.si, _92.cfe, _92.cfs, _92.si, _93.cfe, _93.cfs, _93.si, _94.cfe, _94.cfs, _94.si, _95.cfe, _95.cfs, _95.si, _96.cfe, _96.cfs, _96.si, _97.cfe, _97.cfs, _97.si, _98.cfe, _98.cfs, _98.si, _99.cfe, _99.cfs, _99.si, _9a.cfe, _9a.cfs, _9a.si, _9b.cfe, _9b.cfs, _9b.si, _9c.cfe, _9c.cfs, _9c.si, _9d.cfe, _9d.cfs, _9d.si, _9e.cfe, _9e.cfs, _9e.si, _9f.cfe, _9f.cfs, _9f.si, _9g.cfe, _9g.cfs, _9g.si, _9h.cfe, _9h.cfs, _9h.si, _9i.cfe, _9i.cfs, _9i.si, _9j.cfe, _9j.cfs, _9j.si, _9k.cfe, _9k.cfs, _9k.si, _9l.cfe, _9l.cfs, _9l.si, _9m.cfe, _9m.cfs, _9m.si, _9n.cfe, _9n.cfs, _9n.si, _9o.cfe, _9o.cfs, _9o.si, _9p.cfe, _9p.cfs, _9p.si, _9q.cfe, _9q.cfs, _9q.si, _9r.cfe, _9r.cfs, _9r.si, _9s.cfe, _9s.cfs, _9s.si, _9t.cfe, _9t.cfs, _9t.si, _9u.cfe, _9u.cfs, _9u.si, _9v.cfe, _9v.cfs, _9v.si, _9w.cfe, _9w.cfs, _9w.si, _9x.cfe, _9x.cfs, _9x.si, _9y.cfe, _9y.cfs, _9y.si, _9z.cfe, _9z.cfs, _9z.si, segments_2]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1404642157034,generation=2,filelist=[_90.fdt, _90.fdx, _90.fnm, _90.nvd, _90.nvm, _90.si, _90_Memory_0.ram, _91.cfe, _91.cfs, _91.si, _92.cfe, _92.cfs, _92.si, _93.cfe, _93.cfs, _93.si, _94.cfe, _94.cfs, _94.si, _95.cfe, _95.cfs, _95.si, _96.cfe, _96.cfs, _96.si, _97.cfe, _97.cfs, _97.si, _98.cfe, _98.cfs, _98.si, _99.cfe, _99.cfs, _99.si, _9a.cfe, _9a.cfs, _9a.si, _9b.cfe, _9b.cfs, _9b.si, _9c.cfe, _9c.cfs, _9c.si, _9d.cfe, _9d.cfs, _9d.si, _9e.cfe, _9e.cfs, _9e.si, _9f.cfe, _9f.cfs, _9f.si, _9g.cfe, _9g.cfs, _9g.si, _9h.cfe, _9h.cfs, _9h.si, _9i.cfe, _9i.cfs, _9i.si, _9j.cfe, _9j.cfs, _9j.si, _9k.cfe, _9k.cfs, _9k.si, _9l.cfe, _9l.cfs, _9l.si, _9m.cfe, _9m.cfs, _9m.si, _9n.cfe, _9n.cfs, _9n.si, _9o.cfe, _9o.cfs, _9o.si, _9p.cfe, _9p.cfs, _9p.si, _9q.cfe, _9q.cfs, _9q.si, _9r.cfe, _9r.cfs, _9r.si, _9s.cfe, _9s.cfs, _9s.si, _9t.cfe, _9t.cfs, _9t.si, _9u.cfe, _9u.cfs, _9u.si, _9v.cfe, _9v.cfs, _9v.si, _9w.cfe, _9w.cfs, _9w.si, _9x.cfe, _9x.cfs, _9x.si, _9y.cfe, _9y.cfs, _9y.si, _9z.cfe, _9z.cfs, _9z.si, segments_2]}]> but was:<[{indexVersion=1404642157034,generation=3,filelist=[_a0.fdt, _a0.fdx, _a0.fnm, _a0.nvd, _a0.nvm, _a0.si, _a0_Memory_0.ram, segments_3]}, {indexVersion=1404642157034,generation=2,filelist=[_90.fdt, _90.fdx, _90.fnm, _90.nvd, _90.nvm, _90.si, _90_Memory_0.ram, _91.cfe, _91.cfs, _91.si, _92.cfe, _92.cfs, _92.si, _93.cfe, _93.cfs, _93.si, _94.cfe, _94.cfs, _94.si, _95.cfe, _95.cfs, _95.si, _96.cfe, _96.cfs, _96.si, _97.cfe, _97.cfs, _97.si, _98.cfe, _98.cfs, _98.si, _99.cfe, _99.cfs, _99.si, _9a.cfe, _9a.cfs, _9a.si, _9b.cfe, _9b.cfs, _9b.si, _9c.cfe, _9c.cfs, _9c.si, _9d.cfe, _9d.cfs, _9d.si, _9e.cfe, _9e.cfs, _9e.si, _9f.cfe, _9f.cfs, _9f.si, _9g.cfe, _9g.cfs, _9g.si, _9h.cfe, _9h.cfs, _9h.si, _9i.cfe, _9i.cfs, _9i.si, _9j.cfe, _9j.cfs, _9j.si, _9k.cfe, _9k.cfs, _9k.si, _9l.cfe, _9l.cfs, _9l.si, _9m.cfe, _9m.cfs, _9m.si, _9n.cfe, _9n.cfs, _9n.si, _9o.cfe, _9o.cfs, _9o.si, _9p.cfe, _9p.cfs, _9p.si, _9q.cfe, _9q.cfs, _9q.si, _9r.cfe, _9r.cfs, _9r.si, _9s.cfe, _9s.cfs, _9s.si, _9t.cfe, _9t.cfs, _9t.si, _9u.cfe, _9u.cfs, _9u.si, _9v.cfe, _9v.cfs, _9v.si, _9w.cfe, _9w.cfs, _9w.si, _9x.cfe, _9x.cfs, _9x.si, _9y.cfe, _9y.cfs, _9y.si, _9z.cfe, _9z.cfs, _9z.si, segments_2]}]>
	at __randomizedtesting.SeedInfo.seed([A84134CEA35DC408:8D962FFED315CA0B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1191)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11729 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/init-core-data-001
   [junit4]   2> 811212 T3782 oas.SolrTestCaseJ4.setUp ###Starting testEmptyCommits
   [junit4]   2> 811223 T3782 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 811225 T3782 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:56325
   [junit4]   2> 811226 T3782 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 811226 T3782 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 811226 T3782 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001
   [junit4]   2> 811226 T3782 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/'
   [junit4]   2> 811255 T3782 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/solr.xml
   [junit4]   2> 811309 T3782 oasc.CoreContainer.<init> New CoreContainer 5425050
   [junit4]   2> 811309 T3782 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/]
   [junit4]   2> 811310 T3782 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 811310 T3782 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 811310 T3782 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 811311 T3782 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 811311 T3782 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 811311 T3782 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 811311 T3782 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 811311 T3782 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 811312 T3782 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 811312 T3782 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 811312 T3782 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 811313 T3782 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 811316 T3792 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/'
   [junit4]   2> 811363 T3792 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 811390 T3792 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 811391 T3792 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 811395 T3792 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 811406 T3792 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 811407 T3792 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir ./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/
   [junit4]   2> 811407 T3792 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 811407 T3792 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/, dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data/
   [junit4]   2> 811407 T3792 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 811408 T3792 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data
   [junit4]   2> 811408 T3792 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data/index/
   [junit4]   2> 811408 T3792 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 811408 T3792 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data/index
   [junit4]   2> 811409 T3792 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=4.359375, floorSegmentMB=0.4765625, forceMergeDeletesPctAllowed=12.6715520240664, segmentsPerTier=38.0, maxCFSSegmentSizeMB=1.8388671875, noCFSRatio=0.0
   [junit4]   2> 811409 T3792 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34b69bb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@770e8b29),segFN=segments_1,generation=1}
   [junit4]   2> 811409 T3792 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 811410 T3792 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 811410 T3792 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 811411 T3792 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 811411 T3792 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 811411 T3792 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 811411 T3792 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 811411 T3792 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 811423 T3792 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 811425 T3792 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 811425 T3792 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 811425 T3792 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=87.634765625, floorSegmentMB=0.4609375, forceMergeDeletesPctAllowed=15.109245898055006, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 811425 T3792 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34b69bb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@770e8b29),segFN=segments_1,generation=1}
   [junit4]   2> 811425 T3792 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 811426 T3792 oass.SolrIndexSearcher.<init> Opening Searcher@795a498e[collection1] main
   [junit4]   2> 811426 T3792 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/conf
   [junit4]   2> 811426 T3792 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/conf/}
   [junit4]   2> 811426 T3792 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/conf
   [junit4]   2> 811427 T3792 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
   [junit4]   2> 811427 T3792 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 811427 T3792 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 811427 T3792 oash.ReplicationHandler.inform Replication enabled for following config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 811428 T3792 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 811428 T3793 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@795a498e[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 811428 T3792 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 811428 T3782 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0
   [junit4]   2> 811429 T3782 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 811437 T3782 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 811439 T3782 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:36180
   [junit4]   2> 811439 T3782 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 811439 T3782 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 811439 T3782 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002
   [junit4]   2> 811440 T3782 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/'
   [junit4]   2> 811451 T3782 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/solr.xml
   [junit4]   2> 811479 T3782 oasc.CoreContainer.<init> New CoreContainer 1852671551
   [junit4]   2> 811480 T3782 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/]
   [junit4]   2> 811481 T3782 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 811481 T3782 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 811481 T3782 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 811481 T3782 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 811481 T3782 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 811482 T3782 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 811482 T3782 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 811482 T3782 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 811482 T3782 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 811483 T3782 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 811483 T3782 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 811483 T3782 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 811486 T3804 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/'
   [junit4]   2> 811517 T3804 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 811544 T3804 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 811544 T3804 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 811549 T3804 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 811557 T3804 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 811558 T3804 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir ./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/
   [junit4]   2> 811558 T3804 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 811558 T3804 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/, dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/
   [junit4]   2> 811558 T3804 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 811559 T3804 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data
   [junit4]   2> 811559 T3804 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index/
   [junit4]   2> 811559 T3804 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 811559 T3804 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index
   [junit4]   2> 811560 T3804 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=4.359375, floorSegmentMB=0.4765625, forceMergeDeletesPctAllowed=12.6715520240664, segmentsPerTier=38.0, maxCFSSegmentSizeMB=1.8388671875, noCFSRatio=0.0
   [junit4]   2> 811560 T3804 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19a2df8f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bdb855c),segFN=segments_1,generation=1}
   [junit4]   2> 811560 T3804 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 811561 T3804 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 811561 T3804 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 811561 T3804 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 811562 T3804 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 811562 T3804 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 811562 T3804 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 811562 T3804 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 811563 T3804 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 811565 T3804 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 811565 T3804 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 811565 T3804 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=87.634765625, floorSegmentMB=0.4609375, forceMergeDeletesPctAllowed=15.109245898055006, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 811566 T3804 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19a2df8f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bdb855c),segFN=segments_1,generation=1}
   [junit4]   2> 811566 T3804 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 811566 T3804 oass.SolrIndexSearcher.<init> Opening Searcher@5639cf44[collection1] main
   [junit4]   2> 811566 T3804 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/conf
   [junit4]   2> 811566 T3804 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/conf/}
   [junit4]   2> 811567 T3804 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/conf
   [junit4]   2> 811567 T3804 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
   [junit4]   2> 811567 T3804 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 811567 T3804 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 811572 T3804 oash.SnapPuller.startExecutorService Poll Scheduled at an interval of 1000ms
   [junit4]   2> 811572 T3805 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5639cf44[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 811572 T3804 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 811572 T3782 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0
   [junit4]   2> 811572 T3782 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> ASYNC  NEW_CORE C295 name=collection1 org.apache.solr.core.SolrCore@424f4e0f
   [junit4]   2> 811575 T3785 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={sort=id+desc&wt=javabin&version=2&q=*:*} hits=0 status=0 QTime=1 
   [junit4]   2> 811579 T3785 C295 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[1]} 0 1
   [junit4]   2> 811582 T3785 C295 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 811590 T3785 C295 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34b69bb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@770e8b29),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34b69bb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@770e8b29),segFN=segments_2,generation=2}
   [junit4]   2> 811591 T3785 C295 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 811598 T3785 C295 oass.SolrIndexSearcher.<init> Opening Searcher@7b42aa01[collection1] main
   [junit4]   2> 811598 T3785 C295 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 811598 T3793 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7b42aa01[collection1] main{StandardDirectoryReader(segments_2:3:nrt _0(4.10):c1)}
   [junit4]   2> 811599 T3785 C295 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={commit=true&wt=javabin&version=2} {commit=} 0 17
   [junit4]   2> 811618 T3808 oash.SnapPuller.<init> WARN 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 811618 T3808 oash.SnapPuller.<init>  No value set for 'pollInterval'. Timer Task not started.
   [junit4]   2> 811619 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=indexversion&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811620 T3808 oash.SnapPuller.fetchLatestIndex Master's generation: 2
   [junit4]   2> 811620 T3808 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
   [junit4]   2> 811620 T3808 oash.SnapPuller.fetchLatestIndex Starting replication process
   [junit4]   2> 811627 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={generation=2&command=filelist&wt=javabin&version=2&qt=/replication} status=0 QTime=5 
   [junit4]   2> 811627 T3808 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 4
   [junit4]   2> 811628 T3808 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447382
   [junit4]   2> 811628 T3808 oasu.DefaultSolrCoreState.closeIndexWriter Closing IndexWriter...
   [junit4]   2> 811628 T3808 oasu.DefaultSolrCoreState.closeIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 811628 T3808 oasu.DefaultSolrCoreState.rollbackIndexWriter Rollback old IndexWriter... core=collection1
   [junit4]   2> 811629 T3808 oash.SnapPuller.fetchLatestIndex Starting download to BaseDirectoryWrapper(RAMDirectory@1c00b013 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@310d6341) fullCopy=false
   [junit4]   2> 811632 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=2&command=filecontent&wt=filestream&file=_0.si&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811635 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=2&command=filecontent&wt=filestream&file=segments_2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811640 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=2&command=filecontent&wt=filestream&file=_0.cfe&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811643 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=2&command=filecontent&wt=filestream&file=_0.cfs&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811644 T3808 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
   [junit4]   2> 811655 T3808 oasu.DefaultSolrCoreState.openIndexWriter Creating new IndexWriter...
   [junit4]   2> 811655 T3808 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 811656 T3808 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19a2df8f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bdb855c),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19a2df8f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bdb855c),segFN=segments_2,generation=2}
   [junit4]   2> 811656 T3808 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 811656 T3808 oasu.DefaultSolrCoreState.openIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 811657 T3808 oass.SolrIndexSearcher.<init> Opening Searcher@509658e7[collection1] main
   [junit4]   2> 811657 T3805 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@509658e7[collection1] main{StandardDirectoryReader(segments_2:3:nrt _0(4.10):c1)}
   [junit4]   2> 811657 T3808 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447382 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447382;done=true>>]
   [junit4]   2> 811657 T3808 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447382
   [junit4]   2> ASYNC  NEW_CORE C296 name=collection1 org.apache.solr.core.SolrCore@47468785
   [junit4]   2> 811657 T3797 C296 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={wait=true&masterUrl=http://127.0.0.1:56325/solr/replication&command=fetchindex} status=0 QTime=40 
   [junit4]   2> 811670 T3798 C296 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={sort=id+desc&wt=javabin&version=2&q=name:empty1} hits=1 status=0 QTime=8 
   [junit4]   2> 811674 T3785 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&command=details&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811677 T3785 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&command=details&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811682 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=details&wt=javabin&slave=false&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811682 T3798 C296 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&command=details&wt=javabin&version=2&qt=/replication} status=0 QTime=2 
   [junit4]   2> 811685 T3785 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=assertVersions&command=indexversion&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811688 T3798 C296 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=assertVersions&command=indexversion&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811690 T3785 C295 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 811690 T3785 C295 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 811690 T3785 C295 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 811690 T3785 C295 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 811691 T3785 C295 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={commit=true&wt=javabin&version=2} {commit=} 0 1
   [junit4]   2> 811692 T3811 oash.SnapPuller.<init> WARN 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 811692 T3811 oash.SnapPuller.<init>  No value set for 'pollInterval'. Timer Task not started.
   [junit4]   2> 811694 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=indexversion&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811694 T3811 oash.SnapPuller.fetchLatestIndex Slave in sync with master.
   [junit4]   2> 811694 T3797 C296 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={wait=true&masterUrl=http://127.0.0.1:56325/solr/replication&command=fetchindex} status=0 QTime=2 
   [junit4]   2> 811697 T3785 C295 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[2]} 0 0
   [junit4]   2> 811700 T3785 C295 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 811701 T3785 C295 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34b69bb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@770e8b29),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34b69bb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@770e8b29),segFN=segments_3,generation=3}
   [junit4]   2> 811702 T3785 C295 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 811702 T3785 C295 oass.SolrIndexSearcher.<init> Opening Searcher@39a5a08[collection1] main
   [junit4]   2> 811703 T3785 C295 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 811703 T3793 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@39a5a08[collection1] main{StandardDirectoryReader(segments_3:5:nrt _0(4.10):c1 _1(4.10):c1)}
   [junit4]   2> 811703 T3785 C295 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={commit=true&wt=javabin&version=2} {commit=} 0 3
   [junit4]   2> 811704 T3812 oash.SnapPuller.<init> WARN 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 811704 T3812 oash.SnapPuller.<init>  No value set for 'pollInterval'. Timer Task not started.
   [junit4]   2> 811707 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=indexversion&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811707 T3812 oash.SnapPuller.fetchLatestIndex Master's generation: 3
   [junit4]   2> 811707 T3812 oash.SnapPuller.fetchLatestIndex Slave's generation: 2
   [junit4]   2> 811708 T3812 oash.SnapPuller.fetchLatestIndex Starting replication process
   [junit4]   2> 811711 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={generation=3&command=filelist&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811712 T3812 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 7
   [junit4]   2> 811712 T3812 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447466
   [junit4]   2> 811713 T3812 oasu.DefaultSolrCoreState.closeIndexWriter Closing IndexWriter...
   [junit4]   2> 811713 T3812 oasu.DefaultSolrCoreState.closeIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 811713 T3812 oasu.DefaultSolrCoreState.rollbackIndexWriter Rollback old IndexWriter... core=collection1
   [junit4]   2> 811714 T3812 oash.SnapPuller.fetchLatestIndex Starting download to BaseDirectoryWrapper(RAMDirectory@20c49fd8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5aa43d99) fullCopy=false
   [junit4]   2> 811714 T3812 oash.SnapPuller.downloadIndexFiles Skipping download for _0.si because it already exists
   [junit4]   2> 811717 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=3&command=filecontent&wt=filestream&file=_1.cfe&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811721 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=3&command=filecontent&wt=filestream&file=segments_3&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811722 T3812 oash.SnapPuller.downloadIndexFiles Skipping download for _0.cfe because it already exists
   [junit4]   2> 811722 T3812 oash.SnapPuller.downloadIndexFiles Skipping download for _0.cfs because it already exists
   [junit4]   2> 811725 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=3&command=filecontent&wt=filestream&file=_1.cfs&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811728 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=3&command=filecontent&wt=filestream&file=_1.si&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811729 T3812 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
   [junit4]   2> 811730 T3812 oasu.DefaultSolrCoreState.openIndexWriter Creating new IndexWriter...
   [junit4]   2> 811731 T3812 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 811732 T3812 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19a2df8f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bdb855c),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19a2df8f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bdb855c),segFN=segments_3,generation=3}
   [junit4]   2> 811732 T3812 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 811732 T3812 oasu.DefaultSolrCoreState.openIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 811733 T3812 oass.SolrIndexSearcher.<init> Opening Searcher@53d28106[collection1] main
   [junit4]   2> 811733 T3805 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@53d28106[collection1] main{StandardDirectoryReader(segments_3:5:nrt _0(4.10):c1 _1(4.10):c1)}
   [junit4]   2> 811733 T3812 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447466 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447466;done=true>>]
   [junit4]   2> 811733 T3812 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447466
   [junit4]   2> 811734 T3797 C296 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={wait=true&masterUrl=http://127.0.0.1:56325/solr/replication&command=fetchindex} status=0 QTime=30 
   [junit4]   2> 811738 T3798 C296 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={sort=id+desc&wt=javabin&version=2&q=name:empty2} hits=1 status=0 QTime=1 
   [junit4]   2> 811741 T3785 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&command=details&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811744 T3785 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&command=details&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811749 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=details&wt=javabin&slave=false&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811750 T3798 C296 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&command=details&wt=javabin&version=2&qt=/replication} status=0 QTime=3 
   [junit4]   2> 811753 T3785 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=assertVersions&command=indexversion&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811756 T3798 C296 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=assertVersions&command=indexversion&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811760 T3785 C295 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[3]} 0 1
   [junit4]   2> 811762 T3785 C295 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 811774 T3785 C295 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=3
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34b69bb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@770e8b29),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34b69bb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@770e8b29),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34b69bb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@770e8b29),segFN=segments_4,generation=4}
   [junit4]   2> 811774 T3785 C295 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 811775 T3785 C295 oass.SolrIndexSearcher.<init> Opening Searcher@69036e9e[collection1] realtime
   [junit4]   2> 811775 T3785 C295 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 811776 T3785 C295 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={commit=true&wt=javabin&openSearcher=false&version=2} {commit=} 0 14
   [junit4]   2> 811778 T3814 oash.SnapPuller.<init> WARN 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 811778 T3814 oash.SnapPuller.<init>  No value set for 'pollInterval'. Timer Task not started.
   [junit4]   2> 811781 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=indexversion&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811782 T3814 oash.SnapPuller.fetchLatestIndex Master's generation: 4
   [junit4]   2> 811782 T3814 oash.SnapPuller.fetchLatestIndex Slave's generation: 3
   [junit4]   2> 811782 T3814 oash.SnapPuller.fetchLatestIndex Starting replication process
   [junit4]   2> 811784 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={generation=4&command=filelist&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811785 T3814 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 10
   [junit4]   2> 811785 T3814 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447539
   [junit4]   2> 811786 T3814 oasu.DefaultSolrCoreState.closeIndexWriter Closing IndexWriter...
   [junit4]   2> 811786 T3814 oasu.DefaultSolrCoreState.closeIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 811786 T3814 oasu.DefaultSolrCoreState.rollbackIndexWriter Rollback old IndexWriter... core=collection1
   [junit4]   2> 811787 T3814 oash.SnapPuller.fetchLatestIndex Starting download to BaseDirectoryWrapper(RAMDirectory@159fa4e1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2ddc892) fullCopy=false
   [junit4]   2> 811789 T3786 C295 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=4&command=filecontent&wt=filestream&file=_2.cfe&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811798 T3814 oash.SnapPuller.downloadIndexFiles Skipping download for _0.si because it already exists
   [junit4]   2> ASYNC  NEW_CORE C297 name=collection1 org.apache.solr.core.SolrCore@424f4e0f
   [junit4]   2> 811800 T3786 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=4&command=filecontent&wt=filestream&file=segments_4&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811801 T3814 oash.SnapPuller.downloadIndexFiles Skipping download for _1.cfe because it already exists
   [junit4]   2> 811801 T3814 oash.SnapPuller.downloadIndexFiles Skipping download for _0.cfe because it already exists
   [junit4]   2> 811801 T3814 oash.SnapPuller.downloadIndexFiles Skipping download for _0.cfs because it already exists
   [junit4]   2> 811802 T3814 oash.SnapPuller.downloadIndexFiles Skipping download for _1.cfs because it already exists
   [junit4]   2> 811802 T3814 oash.SnapPuller.downloadIndexFiles Skipping download for _1.si because it already exists
   [junit4]   2> 811804 T3786 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=4&command=filecontent&wt=filestream&file=_2.cfs&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811807 T3786 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=4&command=filecontent&wt=filestream&file=_2.si&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811808 T3814 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
   [junit4]   2> 811809 T3814 oasu.DefaultSolrCoreState.openIndexWriter Creating new IndexWriter...
   [junit4]   2> 811810 T3814 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 811811 T3814 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=3
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19a2df8f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bdb855c),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19a2df8f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bdb855c),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19a2df8f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bdb855c),segFN=segments_4,generation=4}
   [junit4]   2> 811812 T3814 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 811812 T3814 oasu.DefaultSolrCoreState.openIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 811814 T3814 oass.SolrIndexSearcher.<init> Opening Searcher@3285112a[collection1] main
   [junit4]   2> 811814 T3805 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3285112a[collection1] main{StandardDirectoryReader(segments_4:7:nrt _0(4.10):c1 _1(4.10):c1 _2(4.10):c1)}
   [junit4]   2> 811814 T3814 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447539 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447539;done=true>>]
   [junit4]   2> 811815 T3814 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447539
   [junit4]   2> ASYNC  NEW_CORE C298 name=collection1 org.apache.solr.core.SolrCore@47468785
   [junit4]   2> 811815 T3799 C298 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={wait=true&masterUrl=http://127.0.0.1:56325/solr/replication&command=fetchindex} status=0 QTime=37 
   [junit4]   2> 811818 T3785 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={sort=id+desc&wt=javabin&version=2&q=name:empty3} hits=0 status=0 QTime=0 
   [junit4]   2> 811821 T3798 C298 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={sort=id+desc&wt=javabin&version=2&q=name:empty3} hits=1 status=0 QTime=0 
   [junit4]   2> 811825 T3785 C297 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[4]} 0 1
   [junit4]   2> 811828 T3785 C297 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 811829 T3785 C297 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=4
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34b69bb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@770e8b29),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34b69bb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@770e8b29),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34b69bb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@770e8b29),segFN=segments_4,generation=4}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34b69bb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@770e8b29),segFN=segments_5,generation=5}
   [junit4]   2> 811830 T3785 C297 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 5
   [junit4]   2> 811831 T3785 C297 oass.SolrIndexSearcher.<init> Opening Searcher@79639fe5[collection1] main
   [junit4]   2> 811831 T3785 C297 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 811831 T3793 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@79639fe5[collection1] main{StandardDirectoryReader(segments_5:9:nrt _0(4.10):c1 _1(4.10):c1 _2(4.10):c1 _3(4.10):c1)}
   [junit4]   2> 811831 T3785 C297 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={commit=true&wt=javabin&version=2} {commit=} 0 3
   [junit4]   2> 811833 T3816 oash.SnapPuller.<init> WARN 'masterUrl' must be specified without the /replication suffix
   [junit4]   2> 811833 T3816 oash.SnapPuller.<init>  No value set for 'pollInterval'. Timer Task not started.
   [junit4]   2> 811836 T3786 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=indexversion&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811836 T3816 oash.SnapPuller.fetchLatestIndex Master's generation: 5
   [junit4]   2> 811836 T3816 oash.SnapPuller.fetchLatestIndex Slave's generation: 4
   [junit4]   2> 811836 T3816 oash.SnapPuller.fetchLatestIndex Starting replication process
   [junit4]   2> 811839 T3786 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={generation=5&command=filelist&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811839 T3816 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 13
   [junit4]   2> 811840 T3816 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447594
   [junit4]   2> 811840 T3816 oasu.DefaultSolrCoreState.closeIndexWriter Closing IndexWriter...
   [junit4]   2> 811840 T3816 oasu.DefaultSolrCoreState.closeIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 811840 T3816 oasu.DefaultSolrCoreState.rollbackIndexWriter Rollback old IndexWriter... core=collection1
   [junit4]   2> 811841 T3816 oash.SnapPuller.fetchLatestIndex Starting download to BaseDirectoryWrapper(RAMDirectory@3c6cce6f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@577c4075) fullCopy=false
   [junit4]   2> 811844 T3786 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=5&command=filecontent&wt=filestream&file=_3.cfs&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811845 T3816 oash.SnapPuller.downloadIndexFiles Skipping download for _2.cfe because it already exists
   [junit4]   2> 811847 T3786 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=5&command=filecontent&wt=filestream&file=_3.cfe&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811847 T3816 oash.SnapPuller.downloadIndexFiles Skipping download for _1.cfe because it already exists
   [junit4]   2> 811848 T3816 oash.SnapPuller.downloadIndexFiles Skipping download for _0.cfe because it already exists
   [junit4]   2> 811850 T3786 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=5&command=filecontent&wt=filestream&file=_3.si&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811851 T3816 oash.SnapPuller.downloadIndexFiles Skipping download for _1.cfs because it already exists
   [junit4]   2> 811851 T3816 oash.SnapPuller.downloadIndexFiles Skipping download for _1.si because it already exists
   [junit4]   2> 811853 T3786 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={checksum=true&generation=5&command=filecontent&wt=filestream&file=segments_5&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811854 T3816 oash.SnapPuller.downloadIndexFiles Skipping download for _0.si because it already exists
   [junit4]   2> 811854 T3816 oash.SnapPuller.downloadIndexFiles Skipping download for _0.cfs because it already exists
   [junit4]   2> 811854 T3816 oash.SnapPuller.downloadIndexFiles Skipping download for _2.cfs because it already exists
   [junit4]   2> 811854 T3816 oash.SnapPuller.downloadIndexFiles Skipping download for _2.si because it already exists
   [junit4]   2> 811855 T3816 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
   [junit4]   2> 811856 T3816 oasu.DefaultSolrCoreState.openIndexWriter Creating new IndexWriter...
   [junit4]   2> 811857 T3816 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 811859 T3816 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=4
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19a2df8f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bdb855c),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19a2df8f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bdb855c),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19a2df8f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bdb855c),segFN=segments_4,generation=4}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19a2df8f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bdb855c),segFN=segments_5,generation=5}
   [junit4]   2> 811859 T3816 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 5
   [junit4]   2> 811859 T3816 oasu.DefaultSolrCoreState.openIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 811861 T3816 oass.SolrIndexSearcher.<init> Opening Searcher@3b1c4987[collection1] main
   [junit4]   2> 811861 T3805 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3b1c4987[collection1] main{StandardDirectoryReader(segments_5:9:nrt _0(4.10):c1 _1(4.10):c1 _2(4.10):c1 _3(4.10):c1)}
   [junit4]   2> 811862 T3816 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447594 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447594;done=true>>]
   [junit4]   2> 811862 T3816 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index.20140706151447594
   [junit4]   2> 811862 T3800 C298 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={wait=true&masterUrl=http://127.0.0.1:56325/solr/replication&command=fetchindex} status=0 QTime=29 
   [junit4]   2> 811866 T3785 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={sort=id+desc&wt=javabin&version=2&q=name:(empty1+empty2+empty3+empty4)} hits=4 status=0 QTime=1 
   [junit4]   2> 811869 T3798 C298 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={sort=id+desc&wt=javabin&version=2&q=name:(empty1+empty2+empty3+empty4)} hits=4 status=0 QTime=0 
   [junit4]   2> 811872 T3785 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&command=details&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811875 T3785 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&command=details&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811880 T3786 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=details&wt=javabin&slave=false&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811881 T3798 C298 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&command=details&wt=javabin&version=2&qt=/replication} status=0 QTime=3 
   [junit4]   2> 811884 T3785 C297 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=assertVersions&command=indexversion&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811886 T3798 C298 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=assertVersions&command=indexversion&wt=javabin&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 811887 T3782 oas.SolrTestCaseJ4.tearDown ###Ending testEmptyCommits
   [junit4]   2> 811887 T3782 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=5425050
   [junit4]   2> 811888 T3782 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@424f4e0f
   [junit4]   2> 811888 T3782 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=5,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=4,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 811889 T3782 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 811890 T3782 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 811890 T3782 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 811891 T3782 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 811891 T3782 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 811892 T3782 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data;done=false>>]
   [junit4]   2> 811892 T3782 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data
   [junit4]   2> 811893 T3782 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data/index;done=false>>]
   [junit4]   2> 811893 T3782 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data/index
   [junit4]   2> 811894 T3782 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 811948 T3782 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1852671551
   [junit4]   2> 811948 T3782 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@47468785
   [junit4]   2> 811949 T3782 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 811949 T3782 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 811949 T3782 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 811949 T3782 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 811950 T3782 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 811950 T3782 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 811951 T3782 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data;done=false>>]
   [junit4]   2> 811951 T3782 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data
   [junit4]   2> 811951 T3782 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index;done=false>>]
   [junit4]   2> 811951 T3782 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index
   [junit4]   2> 811952 T3782 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 812016 T3782 oas.SolrTestCaseJ4.setUp ###Starting testNoWriter
   [junit4]   2> 812025 T3782 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 812028 T3782 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58063
   [junit4]   2> 812028 T3782 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 812028 T3782 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 812029 T3782 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001
   [junit4]   2> 812029 T3782 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/'
   [junit4]   2> 812058 T3782 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/solr.xml
   [junit4]   2> 812114 T3782 oasc.CoreContainer.<init> New CoreContainer 1331591640
   [junit4]   2> 812114 T3782 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/]
   [junit4]   2> 812115 T3782 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 812115 T3782 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 812116 T3782 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 812116 T3782 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 812116 T3782 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 812116 T3782 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 812116 T3782 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 812117 T3782 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 812117 T3782 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 812117 T3782 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 812118 T3782 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 812118 T3782 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 812121 T3827 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/'
   [junit4]   2> 812157 T3827 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 812187 T3827 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 812187 T3827 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 812192 T3827 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 812204 T3827 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 812205 T3827 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir ./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/
   [junit4]   2> 812205 T3827 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 812205 T3827 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/, dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data/
   [junit4]   2> 812205 T3827 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 812205 T3827 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data
   [junit4]   2> 812206 T3827 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data/index/
   [junit4]   2> 812206 T3827 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 812206 T3827 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data/index
   [junit4]   2> 812206 T3827 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=4.359375, floorSegmentMB=0.4765625, forceMergeDeletesPctAllowed=12.6715520240664, segmentsPerTier=38.0, maxCFSSegmentSizeMB=1.8388671875, noCFSRatio=0.0
   [junit4]   2> 812207 T3827 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1200086c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5a7d8e37),segFN=segments_1,generation=1}
   [junit4]   2> 812207 T3827 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 812208 T3827 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 812208 T3827 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 812208 T3827 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 812208 T3827 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 812209 T3827 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 812209 T3827 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 812209 T3827 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 812221 T3827 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 812222 T3827 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 812223 T3827 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 812223 T3827 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=87.634765625, floorSegmentMB=0.4609375, forceMergeDeletesPctAllowed=15.109245898055006, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 812223 T3827 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1200086c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5a7d8e37),segFN=segments_1,generation=1}
   [junit4]   2> 812223 T3827 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 812224 T3827 oass.SolrIndexSearcher.<init> Opening Searcher@7785924f[collection1] main
   [junit4]   2> 812224 T3827 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/conf
   [junit4]   2> 812224 T3827 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/conf/}
   [junit4]   2> 812224 T3827 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/conf
   [junit4]   2> 812225 T3827 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
   [junit4]   2> 812225 T3827 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 812225 T3827 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 812225 T3827 oash.ReplicationHandler.inform Replication enabled for following config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 812225 T3827 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 812226 T3827 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 812226 T3828 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7785924f[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 812226 T3782 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0
   [junit4]   2> 812226 T3782 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 812237 T3782 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 812240 T3782 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:57182
   [junit4]   2> 812241 T3782 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 812241 T3782 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 812241 T3782 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002
   [junit4]   2> 812242 T3782 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/'
   [junit4]   2> 812258 T3782 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/solr.xml
   [junit4]   2> 812303 T3782 oasc.CoreContainer.<init> New CoreContainer 2487439
   [junit4]   2> 812303 T3782 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/]
   [junit4]   2> 812304 T3782 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 812305 T3782 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 812305 T3782 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 812305 T3782 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 812306 T3782 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 812306 T3782 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 812306 T3782 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 812307 T3782 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 812307 T3782 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 812308 T3782 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 812308 T3782 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 812308 T3782 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 812313 T3839 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/'
   [junit4]   2> 812359 T3839 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 812409 T3839 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 812410 T3839 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 812417 T3839 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 812429 T3839 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 812430 T3839 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir ./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/
   [junit4]   2> 812430 T3839 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 812431 T3839 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/, dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/
   [junit4]   2> 812431 T3839 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 812432 T3839 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data
   [junit4]   2> 812432 T3839 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index/
   [junit4]   2> 812432 T3839 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 812433 T3839 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index
   [junit4]   2> 812433 T3839 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=4.359375, floorSegmentMB=0.4765625, forceMergeDeletesPctAllowed=12.6715520240664, segmentsPerTier=38.0, maxCFSSegmentSizeMB=1.8388671875, noCFSRatio=0.0
   [junit4]   2> 812434 T3839 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7f7ae0ed lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2805fb6a),segFN=segments_1,generation=1}
   [junit4]   2> 812434 T3839 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 812435 T3839 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 812435 T3839 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 812436 T3839 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 812436 T3839 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 812436 T3839 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 812436 T3839 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 812436 T3839 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 812439 T3839 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 812441 T3839 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 812441 T3839 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 812441 T3839 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=17, maxMergedSegmentMB=87.634765625, floorSegmentMB=0.4609375, forceMergeDeletesPctAllowed=15.109245898055006, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 812442 T3839 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7f7ae0ed lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2805fb6a),segFN=segments_1,generation=1}
   [junit4]   2> 812442 T3839 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 812442 T3839 oass.SolrIndexSearcher.<init> Opening Searcher@1270c62c[collection1] main
   [junit4]   2> 812443 T3839 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-

[...truncated too long message...]

=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 1516589 T4650 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C637 name=collection1 org.apache.solr.core.SolrCore@e51aef4
   [junit4]   2> 1516610 T4638 C637 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={sort=id+desc&wt=javabin&version=2&q=*:*} hits=1 status=0 QTime=0 
   [junit4]   2> 1516611 T3782 oas.SolrTestCaseJ4.tearDown ###Ending doTestIndexAndConfigReplication
   [junit4]   2> 1516611 T3782 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1687539383
   [junit4]   2> 1516612 T3782 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6eb08389
   [junit4]   2> 1516612 T3782 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=1,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 1516613 T3782 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1516613 T3782 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1516613 T3782 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1516614 T3782 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1516614 T3782 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1516614 T3782 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data;done=false>>]
   [junit4]   2> 1516615 T3782 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data
   [junit4]   2> 1516615 T3782 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data/index;done=false>>]
   [junit4]   2> 1516615 T3782 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-001/collection1/data/index
   [junit4]   2> 1516616 T3782 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 1516669 T3782 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1538974412
   [junit4]   2> 1516669 T3782 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@e51aef4
   [junit4]   2> 1516670 T3782 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 1516670 T3782 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1516671 T3782 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1516671 T3782 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1516672 T3782 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1516672 T3782 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1516672 T3782 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data;done=false>>]
   [junit4]   2> 1516673 T3782 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data
   [junit4]   2> 1516673 T3782 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index;done=false>>]
   [junit4]   2> 1516674 T3782 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001/solr-instance-002/collection1/data/index
   [junit4]   2> 1516674 T3782 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 1516741 T3782 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandler-A84134CEA35DC408-001
   [junit4]   2> NOTE: test params are: codec=Lucene49: {newname=PostingsFormat(name=Memory doPackFST= false), name=PostingsFormat(name=Memory doPackFST= false), id=PostingsFormat(name=Memory doPackFST= false)}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=mk, timezone=Asia/Tashkent
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=1,free=89045488,total=332922880
   [junit4]   2> NOTE: All tests run in this JVM: [TestAnalyzedSuggestions, ZkCLITest, CoreAdminCreateDiscoverTest, FastVectorHighlighterTest, CoreAdminHandlerTest, TestFoldingMultitermQuery, SimpleFacetsTest, IndexBasedSpellCheckerTest, TestDFRSimilarityFactory, XsltUpdateRequestHandlerTest, TestFreeTextSuggestions, BJQParserTest, DocumentAnalysisRequestHandlerTest, TestRecoveryHdfs, UUIDFieldTest, UpdateParamsTest, DocValuesMultiTest, DocValuesMissingTest, SolrCmdDistributorTest, SuggesterTSTTest, DistributedSpellCheckComponentTest, ChangedSchemaMergeTest, OpenExchangeRatesOrgProviderTest, TestSolrCoreProperties, MinimalSchemaTest, ZkControllerTest, SpellCheckCollatorTest, DeleteReplicaTest, TestStressUserVersions, QueryResultKeyTest, AssignTest, JSONWriterTest, SuggestComponentTest, TestCoreDiscovery, TestCollectionAPI, TestLuceneMatchVersion, SpellPossibilityIteratorTest, DirectUpdateHandlerTest, TestJmxIntegration, CoreAdminRequestStatusTest, StatelessScriptUpdateProcessorFactoryTest, TestSearchPerf, PreAnalyzedUpdateProcessorTest, TestStressReorder, OverseerStatusTest, TestLeaderElectionZkExpiry, OverseerRolesTest, NotRequiredUniqueKeyTest, AliasIntegrationTest, DateFieldTest, RegexBoostProcessorTest, SyncSliceTest, TestReplicationHandler]
   [junit4] Completed on J0 in 705.56s, 13 tests, 1 failure <<< FAILURES!

[...truncated 917 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:474: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/common-build.xml:496: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1299: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:923: There were test failures: 405 suites, 1656 tests, 1 failure, 49 ignored (7 assumptions)

Total time: 220 minutes 16 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-4.x #567
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 440744692 bytes
Compression is 0.0%
Took 1 min 15 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 571 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/571/

2 tests failed.
FAILED:  org.apache.solr.client.solrj.embedded.SolrExampleStreamingBinaryTest.testExampleConfig

Error Message:
Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 {msg=posix_spawn is not a supported process launch mechanism on this platform.,trace=java.lang.Error: posix_spawn is not a supported process launch mechanism on this platform.  at java.lang.UNIXProcess$1.run(UNIXProcess.java:111)  at java.lang.UNIXProcess$1.run(UNIXProcess.java:93)  at java.security.AccessController.doPrivileged(Native Method)  at java.lang.UNIXProcess.&lt;clinit&gt;(UNIXProcess.java:91)  at java.lang.ProcessImpl.start(ProcessImpl.java:130)  at java.lang.ProcessBuilder.start(ProcessBuilder.java:1022)  at java.lang.Runtime.exec(Runtime.java:617)  at java.lang.Runtime.exec(Runtime.java:450)  at java.lang.Runtime.exec(Runtime.java:347)  at org.apache.solr.handler.admin.SystemInfoHandler.execute(SystemInfoHandler.java:220)  at org.apache.solr.handler.admin.SystemInfoHandler.getSystemInfo(SystemInfoHandler.java:176)  at org.apache.solr.handler.admin.SystemInfoHandler.handleRequestBody(SystemInfoHandler.java:97)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)  at org.apache.solr.handler.admin.InfoHandler.handleRequestBody(InfoHandler.java:88)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)  at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:274)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) ,code=500}</title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /solr/admin/info/system. Reason: <pre>    {msg=posix_spawn is not a supported process launch mechanism on this platform.,trace=java.lang.Error: posix_spawn is not a supported process launch mechanism on this platform.  at java.lang.UNIXProcess$1.run(UNIXProcess.java:111)  at java.lang.UNIXProcess$1.run(UNIXProcess.java:93)  at java.security.AccessController.doPrivileged(Native Method)  at java.lang.UNIXProcess.&lt;clinit&gt;(UNIXProcess.java:91)  at java.lang.ProcessImpl.start(ProcessImpl.java:130)  at java.lang.ProcessBuilder.start(ProcessBuilder.java:1022)  at java.lang.Runtime.exec(Runtime.java:617)  at java.lang.Runtime.exec(Runtime.java:450)  at java.lang.Runtime.exec(Runtime.java:347)  at org.apache.solr.handler.admin.SystemInfoHandler.execute(SystemInfoHandler.java:220)  at org.apache.solr.handler.admin.SystemInfoHandler.getSystemInfo(SystemInfoHandler.java:176)  at org.apache.solr.handler.admin.SystemInfoHandler.handleRequestBody(SystemInfoHandler.java:97)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)  at org.apache.solr.handler.admin.InfoHandler.handleRequestBody(InfoHandler.java:88)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)  at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:274)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) ,code=500}</pre></p> <hr /><i><small>Powered by Jetty://</small></i>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 {msg=posix_spawn is not a supported process launch mechanism on this platform.,trace=java.lang.Error: posix_spawn is not a supported process launch mechanism on this platform.
	at java.lang.UNIXProcess$1.run(UNIXProcess.java:111)
	at java.lang.UNIXProcess$1.run(UNIXProcess.java:93)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.lang.UNIXProcess.&lt;clinit&gt;(UNIXProcess.java:91)
	at java.lang.ProcessImpl.start(ProcessImpl.java:130)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1022)
	at java.lang.Runtime.exec(Runtime.java:617)
	at java.lang.Runtime.exec(Runtime.java:450)
	at java.lang.Runtime.exec(Runtime.java:347)
	at org.apache.solr.handler.admin.SystemInfoHandler.execute(SystemInfoHandler.java:220)
	at org.apache.solr.handler.admin.SystemInfoHandler.getSystemInfo(SystemInfoHandler.java:176)
	at org.apache.solr.handler.admin.SystemInfoHandler.handleRequestBody(SystemInfoHandler.java:97)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
	at org.apache.solr.handler.admin.InfoHandler.handleRequestBody(InfoHandler.java:88)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:274)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /solr/admin/info/system. Reason:
<pre>    {msg=posix_spawn is not a supported process launch mechanism on this platform.,trace=java.lang.Error: posix_spawn is not a supported process launch mechanism on this platform.
	at java.lang.UNIXProcess$1.run(UNIXProcess.java:111)
	at java.lang.UNIXProcess$1.run(UNIXProcess.java:93)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.lang.UNIXProcess.&lt;clinit&gt;(UNIXProcess.java:91)
	at java.lang.ProcessImpl.start(ProcessImpl.java:130)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1022)
	at java.lang.Runtime.exec(Runtime.java:617)
	at java.lang.Runtime.exec(Runtime.java:450)
	at java.lang.Runtime.exec(Runtime.java:347)
	at org.apache.solr.handler.admin.SystemInfoHandler.execute(SystemInfoHandler.java:220)
	at org.apache.solr.handler.admin.SystemInfoHandler.getSystemInfo(SystemInfoHandler.java:176)
	at org.apache.solr.handler.admin.SystemInfoHandler.handleRequestBody(SystemInfoHandler.java:97)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
	at org.apache.solr.handler.admin.InfoHandler.handleRequestBody(InfoHandler.java:88)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:274)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([153B2208F0D7F346:A216369037D0159F]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:516)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.client.solrj.SolrExampleTests.testExampleConfig(SolrExampleTests.java:220)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.client.solrj.embedded.SolrExampleStreamingTest.testExampleConfig

Error Message:
Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 {msg=Could not initialize class java.lang.UNIXProcess,trace=java.lang.NoClassDefFoundError: Could not initialize class java.lang.UNIXProcess  at java.lang.ProcessImpl.start(ProcessImpl.java:130)  at java.lang.ProcessBuilder.start(ProcessBuilder.java:1022)  at java.lang.Runtime.exec(Runtime.java:617)  at java.lang.Runtime.exec(Runtime.java:450)  at java.lang.Runtime.exec(Runtime.java:347)  at org.apache.solr.handler.admin.SystemInfoHandler.execute(SystemInfoHandler.java:220)  at org.apache.solr.handler.admin.SystemInfoHandler.getSystemInfo(SystemInfoHandler.java:176)  at org.apache.solr.handler.admin.SystemInfoHandler.handleRequestBody(SystemInfoHandler.java:97)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)  at org.apache.solr.handler.admin.InfoHandler.handleRequestBody(InfoHandler.java:88)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)  at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:274)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) ,code=500}</title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /solr/admin/info/system. Reason: <pre>    {msg=Could not initialize class java.lang.UNIXProcess,trace=java.lang.NoClassDefFoundError: Could not initialize class java.lang.UNIXProcess  at java.lang.ProcessImpl.start(ProcessImpl.java:130)  at java.lang.ProcessBuilder.start(ProcessBuilder.java:1022)  at java.lang.Runtime.exec(Runtime.java:617)  at java.lang.Runtime.exec(Runtime.java:450)  at java.lang.Runtime.exec(Runtime.java:347)  at org.apache.solr.handler.admin.SystemInfoHandler.execute(SystemInfoHandler.java:220)  at org.apache.solr.handler.admin.SystemInfoHandler.getSystemInfo(SystemInfoHandler.java:176)  at org.apache.solr.handler.admin.SystemInfoHandler.handleRequestBody(SystemInfoHandler.java:97)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)  at org.apache.solr.handler.admin.InfoHandler.handleRequestBody(InfoHandler.java:88)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)  at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:274)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) ,code=500}</pre></p> <hr /><i><small>Powered by Jetty://</small></i>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 {msg=Could not initialize class java.lang.UNIXProcess,trace=java.lang.NoClassDefFoundError: Could not initialize class java.lang.UNIXProcess
	at java.lang.ProcessImpl.start(ProcessImpl.java:130)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1022)
	at java.lang.Runtime.exec(Runtime.java:617)
	at java.lang.Runtime.exec(Runtime.java:450)
	at java.lang.Runtime.exec(Runtime.java:347)
	at org.apache.solr.handler.admin.SystemInfoHandler.execute(SystemInfoHandler.java:220)
	at org.apache.solr.handler.admin.SystemInfoHandler.getSystemInfo(SystemInfoHandler.java:176)
	at org.apache.solr.handler.admin.SystemInfoHandler.handleRequestBody(SystemInfoHandler.java:97)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
	at org.apache.solr.handler.admin.InfoHandler.handleRequestBody(InfoHandler.java:88)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:274)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /solr/admin/info/system. Reason:
<pre>    {msg=Could not initialize class java.lang.UNIXProcess,trace=java.lang.NoClassDefFoundError: Could not initialize class java.lang.UNIXProcess
	at java.lang.ProcessImpl.start(ProcessImpl.java:130)
	at java.lang.ProcessBuilder.start(ProcessBuilder.java:1022)
	at java.lang.Runtime.exec(Runtime.java:617)
	at java.lang.Runtime.exec(Runtime.java:450)
	at java.lang.Runtime.exec(Runtime.java:347)
	at org.apache.solr.handler.admin.SystemInfoHandler.execute(SystemInfoHandler.java:220)
	at org.apache.solr.handler.admin.SystemInfoHandler.getSystemInfo(SystemInfoHandler.java:176)
	at org.apache.solr.handler.admin.SystemInfoHandler.handleRequestBody(SystemInfoHandler.java:97)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
	at org.apache.solr.handler.admin.InfoHandler.handleRequestBody(InfoHandler.java:88)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:274)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([153B2208F0D7F346:A216369037D0159F]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:516)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.client.solrj.SolrExampleTests.testExampleConfig(SolrExampleTests.java:220)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 13021 lines...]
   [junit4] Suite: org.apache.solr.client.solrj.embedded.SolrExampleStreamingBinaryTest
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/./solr.client.solrj.embedded.SolrExampleStreamingBinaryTest-153B2208F0D7F346-001/init-core-data-001
   [junit4]   2> 66951 T248 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 66951 T248 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 66952 T248 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 66956 T248 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:31354
   [junit4]   2> 66957 T248 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 66958 T248 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 66958 T248 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr
   [junit4]   2> 66958 T248 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/'
   [junit4]   2> 66998 T248 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/solr.xml
   [junit4]   2> 67080 T248 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr
   [junit4]   2> 67081 T248 oasc.CoreContainer.<init> New CoreContainer 680770415
   [junit4]   2> 67081 T248 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/]
   [junit4]   2> 67083 T248 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 67083 T248 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 67083 T248 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 67084 T248 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 67084 T248 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 67084 T248 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 67085 T248 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 67085 T248 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 67085 T248 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 67086 T248 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 67086 T248 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 67087 T248 oasc.CoreContainer.load Host Name: 
   [junit4]   2> 67090 T248 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr
   [junit4]   2> 67100 T248 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/
   [junit4]   2> 67111 T248 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 67112 T258 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/'
   [junit4]   2> 67151 T258 oasc.SolrConfig.initLibs Adding specified lib dirs to ClassLoader
   [junit4]   2> 67152 T258 oasc.SolrResourceLoader.addToClassLoader WARN No files added to classloader from lib: ../../../contrib/extraction/lib (resolved as: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/../../../contrib/extraction/lib).
   [junit4]   2> 67153 T258 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: ../../../dist/ (resolved as: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/../../../dist).
   [junit4]   2> 67154 T258 oasc.SolrResourceLoader.addToClassLoader WARN No files added to classloader from lib: ../../../contrib/clustering/lib/ (resolved as: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/../../../contrib/clustering/lib).
   [junit4]   2> 67154 T258 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: ../../../dist/ (resolved as: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/../../../dist).
   [junit4]   2> 67155 T258 oasc.SolrResourceLoader.addToClassLoader WARN No files added to classloader from lib: ../../../contrib/langid/lib/ (resolved as: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/../../../contrib/langid/lib).
   [junit4]   2> 67155 T258 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: ../../../dist/ (resolved as: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/../../../dist).
   [junit4]   2> 67156 T258 oasc.SolrResourceLoader.addToClassLoader WARN No files added to classloader from lib: ../../../contrib/velocity/lib (resolved as: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/../../../contrib/velocity/lib).
   [junit4]   2> 67156 T258 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: ../../../dist/ (resolved as: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/../../../dist).
   [junit4]   2> 67200 T258 oasu.SolrIndexConfig.<init> IndexWriter infoStream solr logging is enabled
   [junit4]   2> 67205 T258 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 67288 T258 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 67289 T258 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 67314 T258 oass.IndexSchema.readSchema [collection1] Schema name=example
   [junit4]   2> 67863 T258 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 67865 T258 oasr.RestManager$Registry.registerManagedResource Registered ManagedResource impl org.apache.solr.rest.schema.analysis.ManagedWordSetResource for path /schema/analysis/stopwords/english
   [junit4]   2> 67866 T258 oasr.RestManager$Registry.registerManagedResource Registered ManagedResource impl org.apache.solr.rest.schema.analysis.ManagedSynonymFilterFactory$SynonymManager for path /schema/analysis/synonyms/english
   [junit4]   2> 67895 T258 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 67900 T258 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 67975 T258 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/
   [junit4]   2> 67975 T258 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 67975 T258 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/, dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingBinaryTest-153B2208F0D7F346-001/tempDir-001/
   [junit4]   2> 67975 T258 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6c18906f
   [junit4]   2> 67976 T258 oasc.SolrCore.initListeners [collection1] Added SolrEventListener for newSearcher: org.apache.solr.core.QuerySenderListener{queries=[]}
   [junit4]   2> 67976 T258 oasc.SolrCore.initListeners [collection1] Added SolrEventListener for firstSearcher: org.apache.solr.core.QuerySenderListener{queries=[{q=static firstSearcher warming in solrconfig.xml}]}
   [junit4]   2> 67977 T258 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingBinaryTest-153B2208F0D7F346-001/tempDir-001
   [junit4]   2> 67977 T258 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingBinaryTest-153B2208F0D7F346-001/tempDir-001/index/
   [junit4]   2> 67977 T258 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingBinaryTest-153B2208F0D7F346-001/tempDir-001/index' doesn't exist. Creating new index...
   [junit4]   2> 67978 T258 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingBinaryTest-153B2208F0D7F346-001/tempDir-001/index
   [junit4]   2> 67979 T258 oasu.LoggingInfoStream.message [IFD][coreLoadExecutor-115-thread-1]: init: current segments file is "null"; deletionPolicy=org.apache.solr.core.IndexDeletionPolicyWrapper@4a5cb718
   [junit4]   2> 67979 T258 oasu.LoggingInfoStream.message [IFD][coreLoadExecutor-115-thread-1]: now checkpoint "" [0 segments ; isCommit = false]
   [junit4]   2> 67980 T258 oasu.LoggingInfoStream.message [IFD][coreLoadExecutor-115-thread-1]: 0 msec to checkpoint
   [junit4]   2> 67980 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: init: create=true
   [junit4]   2> 67980 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: 
   [junit4]   2> 	dir=MockDirectoryWrapper(RAMDirectory@76e446df lockFactory=NativeFSLockFactory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingBinaryTest-153B2208F0D7F346-001/tempDir-001/index)
   [junit4]   2> 	index=
   [junit4]   2> 	version=4.10-SNAPSHOT
   [junit4]   2> 	matchVersion=LUCENE_4_10
   [junit4]   2> 	analyzer=null
   [junit4]   2> 	ramBufferSizeMB=100.0
   [junit4]   2> 	maxBufferedDocs=-1
   [junit4]   2> 	maxBufferedDeleteTerms=-1
   [junit4]   2> 	mergedSegmentWarmer=null
   [junit4]   2> 	readerTermsIndexDivisor=1
   [junit4]   2> 	termIndexInterval=32
   [junit4]   2> 	delPolicy=org.apache.solr.core.IndexDeletionPolicyWrapper
   [junit4]   2> 	commit=null
   [junit4]   2> 	openMode=CREATE
   [junit4]   2> 	similarity=org.apache.lucene.search.similarities.DefaultSimilarity
   [junit4]   2> 	mergeScheduler=ConcurrentMergeScheduler: maxThreadCount=1, maxMergeCount=2, mergeThreadPriority=-1
   [junit4]   2> 	default WRITE_LOCK_TIMEOUT=1000
   [junit4]   2> 	writeLockTimeout=1000
   [junit4]   2> 	codec=Lucene49
   [junit4]   2> 	infoStream=org.apache.solr.update.LoggingInfoStream
   [junit4]   2> 	mergePolicy=[TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=5120.0, floorSegmentMB=2.0, forceMergeDeletesPctAllowed=10.0, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 	indexerThreadPool=org.apache.lucene.index.DocumentsWriterPerThreadPool@cf8aecc
   [junit4]   2> 	readerPooling=false
   [junit4]   2> 	perThreadHardLimitMB=1945
   [junit4]   2> 	useCompoundFile=false
   [junit4]   2> 	checkIntegrityAtMerge=false
   [junit4]   2> 	writer=org.apache.lucene.util.SetOnce@1a9bd9bb
   [junit4]   2> 	
   [junit4]   2> 67980 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: now flush at close waitForMerges=true
   [junit4]   2> 67981 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]:   start flush: applyAllDeletes=true
   [junit4]   2> 67981 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]:   index before flush 
   [junit4]   2> 67981 T258 oasu.LoggingInfoStream.message [DW][coreLoadExecutor-115-thread-1]: startFullFlush
   [junit4]   2> 67981 T258 oasu.LoggingInfoStream.message [DW][coreLoadExecutor-115-thread-1]: anyChanges? numDocsInRam=0 deletes=false hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 67981 T258 oasu.LoggingInfoStream.message [DW][coreLoadExecutor-115-thread-1]: coreLoadExecutor-115-thread-1 finishFullFlush success=true
   [junit4]   2> 67982 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: apply all deletes during flush
   [junit4]   2> 67982 T258 oasu.LoggingInfoStream.message [BD][coreLoadExecutor-115-thread-1]: prune sis=null:  minGen=9223372036854775807 packetCount=0
   [junit4]   2> 67982 T258 oasu.LoggingInfoStream.message [CMS][coreLoadExecutor-115-thread-1]: now merge
   [junit4]   2> 67982 T258 oasu.LoggingInfoStream.message [CMS][coreLoadExecutor-115-thread-1]:   index: 
   [junit4]   2> 67982 T258 oasu.LoggingInfoStream.message [CMS][coreLoadExecutor-115-thread-1]:   no more merges pending; now return
   [junit4]   2> 67982 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: waitForMerges
   [junit4]   2> 67983 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: waitForMerges done
   [junit4]   2> 67983 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: now call final commit()
   [junit4]   2> 67983 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: commit: start
   [junit4]   2> 67983 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: commit: enter lock
   [junit4]   2> 67983 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: commit: now prepare
   [junit4]   2> 67984 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: prepareCommit: flush
   [junit4]   2> 67984 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]:   index before flush 
   [junit4]   2> 67984 T258 oasu.LoggingInfoStream.message [DW][coreLoadExecutor-115-thread-1]: startFullFlush
   [junit4]   2> 67984 T258 oasu.LoggingInfoStream.message [DW][coreLoadExecutor-115-thread-1]: anyChanges? numDocsInRam=0 deletes=false hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 67984 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: apply all deletes during flush
   [junit4]   2> 67984 T258 oasu.LoggingInfoStream.message [BD][coreLoadExecutor-115-thread-1]: prune sis=null:  minGen=9223372036854775807 packetCount=0
   [junit4]   2> 67985 T258 oasu.LoggingInfoStream.message [DW][coreLoadExecutor-115-thread-1]: coreLoadExecutor-115-thread-1 finishFullFlush success=true
   [junit4]   2> 67985 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: startCommit(): start
   [junit4]   2> 67985 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: startCommit index= changeCount=1
   [junit4]   2> 67985 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: done all syncs: []
   [junit4]   2> 67985 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: commit: pendingCommit != null
   [junit4]   2> 67986 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: commit: wrote segments file "segments_1"
   [junit4]   2> 67986 T258 oasu.LoggingInfoStream.message [IFD][coreLoadExecutor-115-thread-1]: now checkpoint "" [0 segments ; isCommit = true]
   [junit4]   2> 67986 T258 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76e446df lockFactory=NativeFSLockFactory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingBinaryTest-153B2208F0D7F346-001/tempDir-001/index),segFN=segments_1,generation=1}
   [junit4]   2> 67986 T258 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 67986 T258 oasu.LoggingInfoStream.message [IFD][coreLoadExecutor-115-thread-1]: 0 msec to checkpoint
   [junit4]   2> 67987 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: commit: took 3.2 msec
   [junit4]   2> 67987 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: commit: done
   [junit4]   2> 67987 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: at close: 
   [junit4]   2> 67987 T258 oasc.SolrCore.initWriters created json: solr.JSONResponseWriter
   [junit4]   2> 67988 T258 oasc.SolrCore.initWriters adding lazy queryResponseWriter: solr.VelocityResponseWriter
   [junit4]   2> 67988 T258 oasc.SolrCore.initWriters created velocity: solr.VelocityResponseWriter
   [junit4]   2> 67988 T258 oasc.SolrCore.initWriters created xslt: solr.XSLTResponseWriter
   [junit4]   2> 67988 T258 oasr.XSLTResponseWriter.init xsltCacheLifetimeSeconds=5
   [junit4]   2> 67994 T258 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 67995 T258 oasc.RequestHandlers.initHandlersFromConfig created /select: solr.SearchHandler
   [junit4]   2> 67996 T258 oasc.RequestHandlers.initHandlersFromConfig created /query: solr.SearchHandler
   [junit4]   2> 67996 T258 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 67997 T258 oasc.RequestHandlers.initHandlersFromConfig created /browse: solr.SearchHandler
   [junit4]   2> 67998 T258 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 67998 T258 oasc.RequestHandlers.initHandlersFromConfig created /update/json: solr.UpdateRequestHandler
   [junit4]   2> 67999 T258 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: solr.UpdateRequestHandler
   [junit4]   2> 67999 T258 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.extraction.ExtractingRequestHandler
   [junit4]   2> 68000 T258 oasc.RequestHandlers.initHandlersFromConfig created /update/extract: solr.extraction.ExtractingRequestHandler
   [junit4]   2> 68000 T258 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.FieldAnalysisRequestHandler
   [junit4]   2> 68000 T258 oasc.RequestHandlers.initHandlersFromConfig created /analysis/field: solr.FieldAnalysisRequestHandler
   [junit4]   2> 68000 T258 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.DocumentAnalysisRequestHandler
   [junit4]   2> 68000 T258 oasc.RequestHandlers.initHandlersFromConfig created /analysis/document: solr.DocumentAnalysisRequestHandler
   [junit4]   2> 68001 T258 oasc.RequestHandlers.initHandlersFromConfig created /admin/: solr.admin.AdminHandlers
   [junit4]   2> 68001 T258 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: solr.PingRequestHandler
   [junit4]   2> 68002 T258 oasc.RequestHandlers.initHandlersFromConfig created /debug/dump: solr.DumpRequestHandler
   [junit4]   2> 68003 T258 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 68003 T258 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.SearchHandler
   [junit4]   2> 68003 T258 oasc.RequestHandlers.initHandlersFromConfig created /spell: solr.SearchHandler
   [junit4]   2> 68003 T258 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.SearchHandler
   [junit4]   2> 68004 T258 oasc.RequestHandlers.initHandlersFromConfig created /suggest: solr.SearchHandler
   [junit4]   2> 68004 T258 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.SearchHandler
   [junit4]   2> 68004 T258 oasc.RequestHandlers.initHandlersFromConfig created /tvrh: solr.SearchHandler
   [junit4]   2> 68004 T258 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.SearchHandler
   [junit4]   2> 68005 T258 oasc.RequestHandlers.initHandlersFromConfig created /terms: solr.SearchHandler
   [junit4]   2> 68005 T258 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.SearchHandler
   [junit4]   2> 68005 T258 oasc.RequestHandlers.initHandlersFromConfig created /elevate: solr.SearchHandler
   [junit4]   2> 68017 T258 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 68018 T258 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 68020 T258 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 68024 T258 oasu.CommitTracker.<init> Hard AutoCommit: if uncommited for 15000ms; 
   [junit4]   2> 68024 T258 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 68026 T258 oasu.LoggingInfoStream.message [IFD][coreLoadExecutor-115-thread-1]: init: current segments file is "segments_1"; deletionPolicy=org.apache.solr.core.IndexDeletionPolicyWrapper@4a5cb718
   [junit4]   2> 68026 T258 oasu.LoggingInfoStream.message [IFD][coreLoadExecutor-115-thread-1]: init: load commit "segments_1"
   [junit4]   2> 68026 T258 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76e446df lockFactory=NativeFSLockFactory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingBinaryTest-153B2208F0D7F346-001/tempDir-001/index),segFN=segments_1,generation=1}
   [junit4]   2> 68026 T258 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 68027 T258 oasu.LoggingInfoStream.message [IFD][coreLoadExecutor-115-thread-1]: now checkpoint "" [0 segments ; isCommit = false]
   [junit4]   2> 68027 T258 oasu.LoggingInfoStream.message [IFD][coreLoadExecutor-115-thread-1]: 0 msec to checkpoint
   [junit4]   2> 68027 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: init: create=false
   [junit4]   2> 68027 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: 
   [junit4]   2> 	dir=MockDirectoryWrapper(RAMDirectory@76e446df lockFactory=NativeFSLockFactory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingBinaryTest-153B2208F0D7F346-001/tempDir-001/index)
   [junit4]   2> 	index=
   [junit4]   2> 	version=4.10-SNAPSHOT
   [junit4]   2> 	matchVersion=LUCENE_4_10
   [junit4]   2> 	analyzer=null
   [junit4]   2> 	ramBufferSizeMB=100.0
   [junit4]   2> 	maxBufferedDocs=-1
   [junit4]   2> 	maxBufferedDeleteTerms=-1
   [junit4]   2> 	mergedSegmentWarmer=null
   [junit4]   2> 	readerTermsIndexDivisor=1
   [junit4]   2> 	termIndexInterval=32
   [junit4]   2> 	delPolicy=org.apache.solr.core.IndexDeletionPolicyWrapper
   [junit4]   2> 	commit=null
   [junit4]   2> 	openMode=APPEND
   [junit4]   2> 	similarity=org.apache.lucene.search.similarities.DefaultSimilarity
   [junit4]   2> 	mergeScheduler=ConcurrentMergeScheduler: maxThreadCount=1, maxMergeCount=2, mergeThreadPriority=-1
   [junit4]   2> 	default WRITE_LOCK_TIMEOUT=1000
   [junit4]   2> 	writeLockTimeout=1000
   [junit4]   2> 	codec=Lucene49
   [junit4]   2> 	infoStream=org.apache.solr.update.LoggingInfoStream
   [junit4]   2> 	mergePolicy=[TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=5120.0, floorSegmentMB=2.0, forceMergeDeletesPctAllowed=10.0, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 	indexerThreadPool=org.apache.lucene.index.DocumentsWriterPerThreadPool@67bfbf22
   [junit4]   2> 	readerPooling=false
   [junit4]   2> 	perThreadHardLimitMB=1945
   [junit4]   2> 	useCompoundFile=false
   [junit4]   2> 	checkIntegrityAtMerge=false
   [junit4]   2> 	writer=org.apache.lucene.util.SetOnce@5d9d8664
   [junit4]   2> 	
   [junit4]   2> 68028 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: flush at getReader
   [junit4]   2> 68028 T258 oasu.LoggingInfoStream.message [DW][coreLoadExecutor-115-thread-1]: startFullFlush
   [junit4]   2> 68028 T258 oasu.LoggingInfoStream.message [DW][coreLoadExecutor-115-thread-1]: anyChanges? numDocsInRam=0 deletes=false hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 68028 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: apply all deletes during flush
   [junit4]   2> 68028 T258 oasu.LoggingInfoStream.message [BD][coreLoadExecutor-115-thread-1]: prune sis=segments_1:  minGen=9223372036854775807 packetCount=0
   [junit4]   2> 68029 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: return reader version=1 reader=StandardDirectoryReader(segments_1:1:nrt)
   [junit4]   2> 68029 T258 oasu.LoggingInfoStream.message [DW][coreLoadExecutor-115-thread-1]: coreLoadExecutor-115-thread-1 finishFullFlush success=true
   [junit4]   2> 68029 T258 oasu.LoggingInfoStream.message [IW][coreLoadExecutor-115-thread-1]: getReader took 1 msec
   [junit4]   2> 68029 T258 oass.SolrIndexSearcher.<init> Opening Searcher@75a68b68[collection1] main
   [junit4]   2> 68030 T258 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/conf
   [junit4]   2> 68030 T258 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/conf/}
   [junit4]   2> 68030 T258 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/conf
   [junit4]   2> 68031 T258 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
   [junit4]   2> 68031 T258 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 68031 T258 oasr.RestManager.init Initializing 2 registered ManagedResources
   [junit4]   2> 68031 T258 oasr.ManagedResourceStorage.load Reading _schema_analysis_stopwords_english.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/conf
   [junit4]   2> 68032 T258 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _schema_analysis_stopwords_english.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/conf
   [junit4]   2> 68032 T258 oasr.ManagedResource.reloadFromStorage Loaded initArgs {ignoreCase=true} for /schema/analysis/stopwords/english
   [junit4]   2> 68032 T258 oasrsa.ManagedWordSetResource.onManagedDataLoadedFromStorage Loaded 35 words for /schema/analysis/stopwords/english
   [junit4]   2> 68032 T258 oasr.ManagedResource.notifyObserversDuringInit Notified 1 observers of /schema/analysis/stopwords/english
   [junit4]   2> 68033 T258 oasr.ManagedResourceStorage.load Reading _schema_analysis_synonyms_english.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/conf
   [junit4]   2> 68033 T258 oasr.ManagedResourceStorage.load Loaded LinkedHashMap at path _schema_analysis_synonyms_english.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/conf
   [junit4]   2> 68033 T258 oasr.ManagedResource.reloadFromStorage Loaded initArgs {ignoreCase=true,format=solr} for /schema/analysis/synonyms/english
   [junit4]   2> 68033 T258 oasrsa.ManagedSynonymFilterFactory$SynonymManager.onManagedDataLoadedFromStorage Loaded 3 synonym mappings for /schema/analysis/synonyms/english
   [junit4]   2> 68034 T258 oasr.ManagedResource.notifyObserversDuringInit Notified 1 observers of /schema/analysis/synonyms/english
   [junit4]   2> 68034 T258 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 68034 T258 oass.DirectSolrSpellChecker.init init: {name=default,field=text,classname=solr.DirectSolrSpellChecker,distanceMeasure=internal,accuracy=0.5,maxEdits=2,minPrefix=1,maxInspections=5,minQueryLength=4,maxQueryFrequency=0.01}
   [junit4]   2> 68035 T258 oashc.SpellCheckComponent.inform No queryConverter defined, using default converter
   [junit4]   2> 68035 T258 oashc.SuggestComponent.inform Initializing SuggestComponent
   [junit4]   2> 68035 T258 oasss.SolrSuggester.init init: {name=mySuggester,lookupImpl=FuzzyLookupFactory,dictionaryImpl=DocumentDictionaryFactory,field=cat,weightField=price,suggestAnalyzerFieldType=string}
   [junit4]   2> 68036 T258 oasss.SolrSuggester.init Dictionary loaded with params: {name=mySuggester,lookupImpl=FuzzyLookupFactory,dictionaryImpl=DocumentDictionaryFactory,field=cat,weightField=price,suggestAnalyzerFieldType=string}
   [junit4]   2> 68036 T258 oashc.QueryElevationComponent.inform Loading QueryElevation from: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/example/solr/collection1/conf/elevate.xml
   [junit4]   2> 68072 T258 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 68072 T259 oasc.QuerySenderListener.newSearcher QuerySenderListener sending requests to Searcher@75a68b68[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> ASYNC  NEW_CORE C19 name=collection1 org.apache.solr.core.SolrCore@7f9de19a
   [junit4]   2> 68074 T259 C19 oasc.SolrCore.execute [collection1] webapp=null path=null params={q=static+firstSearcher+warming+in+solrconfig.xml&event=firstSearcher&distrib=false} hits=0 status=0 QTime=1 
   [junit4]   2> 68074 T259 oasc.QuerySenderListener.newSearcher QuerySenderListener done.
   [junit4]   2> 68074 T259 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 68074 T259 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 68074 T259 oashc.SuggestComponent$SuggesterListener.newSearcher Loading suggester index for: mySuggester
   [junit4]   2> 68074 T259 oasss.SolrSuggester.reload reload()
   [junit4]   2> 68075 T259 oasss.SolrSuggester.build build()
   [junit4]   2> 68080 T259 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@75a68b68[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 68083 T258 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 68083 T248 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0
   [junit4]   2> 68084 T248 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 68084 T248 oas.SolrJettyTestBase.createJetty Jetty Assigned Port#31354
   [junit4]   2> 68092 T248 oas.SolrTestCaseJ4.setUp ###Starting testFaceting
   [junit4]   2> 68097 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: anyChanges? numDocsInRam=0 deletes=true hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 68098 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: nrtIsCurrent: infoVersion matches: true; DW changes: true; BD changes: false
   [junit4]   2> 68098 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: anyChanges? numDocsInRam=0 deletes=true hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 68098 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: flush at getReader
   [junit4]   2> 68099 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: startFullFlush
   [junit4]   2> 68099 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: anyChanges? numDocsInRam=0 deletes=true hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 68099 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: qtp386453935-251: flush naked frozen global deletes
   [junit4]   2> 68100 T251 C19 oasu.LoggingInfoStream.message [BD][qtp386453935-251]: push deletes  1 deleted queries bytesUsed=64 delGen=2 packetCount=1 totBytesUsed=64
   [junit4]   2> 68100 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: flush: push buffered updates:  1 deleted queries bytesUsed=64
   [junit4]   2> 68101 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: apply all deletes during flush
   [junit4]   2> 68101 T251 C19 oasu.LoggingInfoStream.message [BD][qtp386453935-251]: prune sis=segments_1:  minGen=9223372036854775807 packetCount=1
   [junit4]   2> 68101 T251 C19 oasu.LoggingInfoStream.message [BD][qtp386453935-251]: pruneDeletes: prune 1 packets; 0 packets remain
   [junit4]   2> 68102 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: return reader version=1 reader=StandardDirectoryReader(segments_1:1:nrt)
   [junit4]   2> 68102 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: qtp386453935-251 finishFullFlush success=true
   [junit4]   2> 68102 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: getReader took 4 msec
   [junit4]   2> 68103 T251 C19 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {deleteByQuery=*:* (-1472798803623936000)} 0 7
   [junit4]   2> 68106 T251 C19 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 68107 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: anyChanges? numDocsInRam=0 deletes=false hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 68107 T251 C19 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 68107 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: anyChanges? numDocsInRam=0 deletes=false hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 68108 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: nrtIsCurrent: infoVersion matches: true; DW changes: false; BD changes: false
   [junit4]   2> 68108 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: anyChanges? numDocsInRam=0 deletes=false hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 68108 T251 C19 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 68108 T251 C19 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 68109 T251 C19 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&waitSearcher=true&softCommit=false&commit=true&wt=javabin} {commit=} 0 3
   [junit4]   2> 68113 T251 C19 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={q=*:*&version=2&wt=javabin} hits=0 status=0 QTime=1 
   [junit4]   2> 68368 T251 C19 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[1 (1472798803647004672), 2 (1472798803650150400), 3 (1472798803651198976), 4 (1472798803651198977), 5 (1472798803651198978), 6 (1472798803652247552), 7 (1472798803652247553), 8 (1472798803652247554), 9 (1472798803653296128), 10 (1472798803653296129)]} 0 251
   [junit4]   2> 68372 T251 C19 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 68373 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: commit: start
   [junit4]   2> 68373 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: commit: enter lock
   [junit4]   2> 68373 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: commit: now prepare
   [junit4]   2> 68373 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: prepareCommit: flush
   [junit4]   2> 68374 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]:   index before flush 
   [junit4]   2> 68374 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: startFullFlush
   [junit4]   2> 68374 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: anyChanges? numDocsInRam=10 deletes=true hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 68374 T251 C19 oasu.LoggingInfoStream.message [DWFC][qtp386453935-251]: addFlushableState DocumentsWriterPerThread [pendingDeletes=gen=0 9 deleted terms (unique count=9) bytesUsed=1486, segment=_0, aborting=false, numDocsInRAM=10, deleteQueue=DWDQ: [ generation: 2 ]]
   [junit4]   2> 68375 T251 C19 oasu.LoggingInfoStream.message [DWPT][qtp386453935-251]: flush postings as segment _0 numDocs=10
   [junit4]   2> 68377 T251 C19 oasu.LoggingInfoStream.message [DWPT][qtp386453935-251]: new segment has 0 deleted docs
   [junit4]   2> 68378 T251 C19 oasu.LoggingInfoStream.message [DWPT][qtp386453935-251]: new segment has no vectors; norms; no docValues; prox; freqs
   [junit4]   2> 68378 T251 C19 oasu.LoggingInfoStream.message [DWPT][qtp386453935-251]: flushedFiles=[_0.fnm, _0_Lucene41_0.tip, _0.fdx, _0.nvm, _0.fdt, _0_Lucene41_0.doc, _0_Lucene41_0.pos, _0.nvd, _0_Lucene41_0.tim]
   [junit4]   2> 68378 T251 C19 oasu.LoggingInfoStream.message [DWPT][qtp386453935-251]: flushed codec=Lucene49
   [junit4]   2> 68379 T251 C19 oasu.LoggingInfoStream.message [DWPT][qtp386453935-251]: flushed: segment=_0 ramUsed=0.081 MB newFlushedSize(includes docstores)=0.002 MB docs/MB=6,256.42
   [junit4]   2> 68379 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: publishFlushedSegment seg-private updates=null
   [junit4]   2> 68379 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: publishFlushedSegment
   [junit4]   2> 68380 T251 C19 oasu.LoggingInfoStream.message [BD][qtp386453935-251]: push deletes  10 deleted terms (unique count=10) bytesUsed=1056 delGen=4 packetCount=1 totBytesUsed=1056
   [junit4]   2> 68380 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: publish sets newSegment delGen=5 seg=_0(4.10):C10
   [junit4]   2> 68380 T251 C19 oasu.LoggingInfoStream.message [IFD][qtp386453935-251]: now checkpoint "_0(4.10):C10" [1 segments ; isCommit = false]
   [junit4]   2> 68380 T251 C19 oasu.LoggingInfoStream.message [IFD][qtp386453935-251]: 0 msec to checkpoint
   [junit4]   2> 68381 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: apply all deletes during flush
   [junit4]   2> 68381 T251 C19 oasu.LoggingInfoStream.message [BD][qtp386453935-251]: applyDeletes: infos=[_0(4.10):C10] packetCount=1
   [junit4]   2> 68381 T251 C19 oasu.LoggingInfoStream.message [BD][qtp386453935-251]: applyDeletes took 0 msec
   [junit4]   2> 68381 T251 C19 oasu.LoggingInfoStream.message [BD][qtp386453935-251]: prune sis=segments_1: _0(4.10):C10 minGen=6 packetCount=1
   [junit4]   2> 68382 T251 C19 oasu.LoggingInfoStream.message [BD][qtp386453935-251]: pruneDeletes: prune 1 packets; 0 packets remain
   [junit4]   2> 68382 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: qtp386453935-251 finishFullFlush success=true
   [junit4]   2> 68382 T251 C19 oasu.LoggingInfoStream.message [TMP][qtp386453935-251]: findMerges: 1 segments
   [junit4]   2> 68383 T251 C19 oasu.LoggingInfoStream.message [TMP][qtp386453935-251]:   seg=_0(4.10):C10 size=0.001 MB [floored]
   [junit4]   2> 68383 T251 C19 oasu.LoggingInfoStream.message [TMP][qtp386453935-251]:   allowedSegmentCount=1 vs count=1 (eligible count=1) tooBigCount=0
   [junit4]   2> 68383 T251 C19 oasu.LoggingInfoStream.message [CMS][qtp386453935-251]: now merge
   [junit4]   2> 68384 T251 C19 oasu.LoggingInfoStream.message [CMS][qtp386453935-251]:   index: _0(4.10):C10
   [junit4]   2> 68384 T251 C19 oasu.LoggingInfoStream.message [CMS][qtp386453935-251]:   no more merges pending; now return
   [junit4]   2> 68384 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: startCommit(): start
   [junit4]   2> 68384 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: startCommit index=_0(4.10):C10 changeCount=3
   [junit4]   2> 68385 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: done all syncs: [_0.fnm, _0_Lucene41_0.tip, _0.fdx, _0.nvm, _0.fdt, _0.si, _0_Lucene41_0.doc, _0.nvd, _0_Lucene41_0.pos, _0_Lucene41_0.tim]
   [junit4]   2> 68385 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: commit: pendingCommit != null
   [junit4]   2> 68385 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: commit: wrote segments file "segments_2"
   [junit4]   2> 68386 T251 C19 oasu.LoggingInfoStream.message [IFD][qtp386453935-251]: now checkpoint "_0(4.10):C10" [1 segments ; isCommit = true]
   [junit4]   2> 68386 T251 C19 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76e446df lockFactory=NativeFSLockFactory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingBinaryTest-153B2208F0D7F346-001/tempDir-001/index),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76e446df lockFactory=NativeFSLockFactory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingBinaryTest-153B2208F0D7F346-001/tempDir-001/index),segFN=segments_2,generation=2}
   [junit4]   2> 68386 T251 C19 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 68387 T251 C19 oasu.LoggingInfoStream.message [IFD][qtp386453935-251]: deleteCommits: now decRef commit "segments_1"
   [junit4]   2> 68387 T251 C19 oasu.LoggingInfoStream.message [IFD][qtp386453935-251]: delete "segments_1"
   [junit4]   2> 68387 T251 C19 oasu.LoggingInfoStream.message [IFD][qtp386453935-251]: 1 msec to checkpoint
   [junit4]   2> 68388 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: commit: took 14.2 msec
   [junit4]   2> 68388 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: commit: done
   [junit4]   2> 68388 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: anyChanges? numDocsInRam=0 deletes=false hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 68388 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: nrtIsCurrent: infoVersion matches: false; DW changes: false; BD changes: false
   [junit4]   2> 68389 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: flush at getReader
   [junit4]   2> 68389 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: startFullFlush
   [junit4]   2> 68389 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: anyChanges? numDocsInRam=0 deletes=false hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 68390 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: apply all deletes during flush
   [junit4]   2> 68390 T251 C19 oasu.LoggingInfoStream.message [BD][qtp386453935-251]: applyDeletes: no deletes; skipping
   [junit4]   2> 68390 T251 C19 oasu.LoggingInfoStream.message [BD][qtp386453935-251]: prune sis=segments_2: _0(4.10):C10 minGen=6 packetCount=0
   [junit4]   2> 68391 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: return reader version=3 reader=StandardDirectoryReader(segments_2:3:nrt _0(4.10):C10)
   [junit4]   2> 68392 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: qtp386453935-251 finishFullFlush success=true
   [junit4]   2> 68392 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: getReader took 3 msec
   [junit4]   2> 68392 T251 C19 oass.SolrIndexSearcher.<init> Opening Searcher@442b578f[collection1] main
   [junit4]   2> 68393 T251 C19 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 68393 T259 oasc.QuerySenderListener.newSearcher QuerySenderListener sending requests to Searcher@442b578f[collection1] main{StandardDirectoryReader(segments_2:3:nrt _0(4.10):C10)}
   [junit4]   2> 68393 T259 oasc.QuerySenderListener.newSearcher QuerySenderListener done.
   [junit4]   2> 68397 T259 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@442b578f[collection1] main{StandardDirectoryReader(segments_2:3:nrt _0(4.10):C10)}
   [junit4]   2> 68398 T251 C19 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&waitSearcher=true&softCommit=false&commit=true&wt=javabin} {commit=} 0 26
   [junit4]   2> 68402 T251 C19 oasr.UnInvertedField.<init> UnInverted multi-valued field {field=features,memSize=4564,tindexSize=44,time=0,phase1=0,nTerms=4,bigTerms=2,termInstances=4,uses=0}
   [junit4]   2> 68403 T251 C19 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={facet.field=features&q=*:*&facet.mincount=0&version=2&rows=0&facet=true&wt=javabin} hits=10 status=0 QTime=2 
   [junit4]   2> 68407 T251 C19 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={facet.field=features&q=*:*&facet.mincount=0&version=2&rows=0&facet=true&fq=features:two&wt=javabin} hits=5 status=0 QTime=1 
   [junit4]   2> 68411 T251 C19 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={facet.field=features&q=*:*&facet.mincount=4&version=2&rows=0&facet=true&fq=features:two&wt=javabin} hits=5 status=0 QTime=0 
   [junit4]   2> 68415 T251 C19 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={facet.field=features&q=*:*&facet.mincount=-1&version=2&rows=0&facet=true&fq=features:two&wt=javabin} hits=5 status=0 QTime=1 
   [junit4]   2> 68415 T248 oas.SolrTestCaseJ4.tearDown ###Ending testFaceting
   [junit4]   2> 68424 T248 oas.SolrTestCaseJ4.setUp ###Starting testChineseDefaults
   [junit4]   2> 68429 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: anyChanges? numDocsInRam=0 deletes=true hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 68430 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: nrtIsCurrent: infoVersion matches: true; DW changes: true; BD changes: false
   [junit4]   2> 68430 T251 C19 oasu.LoggingInfoStream.message [DW][qtp386453935-251]: anyChanges? numDocsInRam=0 deletes=true hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 68430 T251 C19 oasu.LoggingInfoStream.message [IW][qtp386453935-251]: flush at getReader
   [junit4]   2> 68431 T251 C19 oasu.LoggingInfoStream.messa

[...truncated too long message...]

F346]-worker]: waitForMerges
   [junit4]   2> 113879 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: waitForMerges done
   [junit4]   2> 113879 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: now flush at close waitForMerges=true
   [junit4]   2> 113879 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]:   start flush: applyAllDeletes=true
   [junit4]   2> 113879 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]:   index before flush _2g(4.10):C1 _2h(4.10):C1
   [junit4]   2> 113880 T277 C24 oasu.LoggingInfoStream.message [DW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: startFullFlush
   [junit4]   2> 113880 T277 C24 oasu.LoggingInfoStream.message [DW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: anyChanges? numDocsInRam=0 deletes=false hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 113881 T277 C24 oasu.LoggingInfoStream.message [DW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker finishFullFlush success=true
   [junit4]   2> 113881 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: apply all deletes during flush
   [junit4]   2> 113881 T277 C24 oasu.LoggingInfoStream.message [BD][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: applyDeletes: no deletes; skipping
   [junit4]   2> 113882 T277 C24 oasu.LoggingInfoStream.message [BD][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: prune sis=segments_2b: _2g(4.10):C1 _2h(4.10):C1 minGen=396 packetCount=0
   [junit4]   2> 113882 T277 C24 oasu.LoggingInfoStream.message [CMS][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: now merge
   [junit4]   2> 113882 T277 C24 oasu.LoggingInfoStream.message [CMS][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]:   index: _2g(4.10):C1 _2h(4.10):C1
   [junit4]   2> 113883 T277 C24 oasu.LoggingInfoStream.message [CMS][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]:   no more merges pending; now return
   [junit4]   2> 113883 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: waitForMerges
   [junit4]   2> 113883 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: waitForMerges done
   [junit4]   2> 113884 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: now call final commit()
   [junit4]   2> 113884 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: commit: start
   [junit4]   2> 113884 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: commit: enter lock
   [junit4]   2> 113885 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: commit: now prepare
   [junit4]   2> 113885 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: prepareCommit: flush
   [junit4]   2> 113885 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]:   index before flush _2g(4.10):C1 _2h(4.10):C1
   [junit4]   2> 113886 T277 C24 oasu.LoggingInfoStream.message [DW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: startFullFlush
   [junit4]   2> 113886 T277 C24 oasu.LoggingInfoStream.message [DW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: anyChanges? numDocsInRam=0 deletes=false hasTickets:false pendingChangesInFullFlush: false
   [junit4]   2> 113886 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: apply all deletes during flush
   [junit4]   2> 113887 T277 C24 oasu.LoggingInfoStream.message [BD][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: applyDeletes: no deletes; skipping
   [junit4]   2> 113887 T277 C24 oasu.LoggingInfoStream.message [BD][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: prune sis=segments_2b: _2g(4.10):C1 _2h(4.10):C1 minGen=396 packetCount=0
   [junit4]   2> 113887 T277 C24 oasu.LoggingInfoStream.message [DW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker finishFullFlush success=true
   [junit4]   2> 113888 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: startCommit(): start
   [junit4]   2> 113888 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]:   skip startCommit(): no changes pending
   [junit4]   2> 113888 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: commit: pendingCommit == null; skip
   [junit4]   2> 113889 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: commit: done
   [junit4]   2> 113889 T277 C24 oasu.LoggingInfoStream.message [IW][SUITE-SolrExampleStreamingTest-seed#[153B2208F0D7F346]-worker]: at close: _2g(4.10):C1 _2h(4.10):C1
   [junit4]   2> 113890 T277 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 113891 T277 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 113891 T277 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingTest-153B2208F0D7F346-001/tempDir-001 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingTest-153B2208F0D7F346-001/tempDir-001;done=false>>]
   [junit4]   2> 113892 T277 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingTest-153B2208F0D7F346-001/tempDir-001
   [junit4]   2> 113892 T277 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingTest-153B2208F0D7F346-001/tempDir-001/index [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingTest-153B2208F0D7F346-001/tempDir-001/index;done=false>>]
   [junit4]   2> 113892 T277 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/solr.client.solrj.embedded.SolrExampleStreamingTest-153B2208F0D7F346-001/tempDir-001/index
   [junit4]   2> 113893 T277 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 113947 T277 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-solrj/test/J0/./solr.client.solrj.embedded.SolrExampleStreamingTest-153B2208F0D7F346-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene49), sim=DefaultSimilarity, locale=tr_TR, timezone=America/Monterrey
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=1,free=156817904,total=287834112
   [junit4]   2> NOTE: All tests run in this JVM: [ClientUtilsTest, JettyWebappTest, TestUpdateRequestCodec, AnlysisResponseBaseTest, ModifiableSolrParamsTest, SolrExceptionTest, TestEmbeddedSolrServer, TermsResponseTest, SolrExampleEmbeddedTest, CloudSolrServerTest, LargeVolumeJettyTest, SolrExampleStreamingBinaryTest, SolrExampleStreamingTest]
   [junit4] Completed on J0 in 22.52s, 28 tests, 1 error <<< FAILURES!

[...truncated 53 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:474: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build.xml:202: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/common-build.xml:496: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1299: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:923: There were test failures: 49 suites, 295 tests, 2 errors

Total time: 208 minutes 45 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-4.x #567
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 11526885 bytes
Compression is 0.0%
Took 3.3 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 570 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/570/

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandlerBackup

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.handler.TestReplicationHandlerBackup:     1) Thread[id=18982, name=Thread-4347, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]         at java.net.PlainSocketImpl.socketConnect(Native Method)         at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)         at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)         at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)         at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)         at java.net.Socket.connect(Socket.java:579)         at java.net.Socket.connect(Socket.java:528)         at sun.net.NetworkClient.doConnect(NetworkClient.java:180)         at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)         at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)         at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)         at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)         at java.net.URL.openStream(URL.java:1037)         at org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.TestReplicationHandlerBackup: 
   1) Thread[id=18982, name=Thread-4347, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]
        at java.net.PlainSocketImpl.socketConnect(Native Method)
        at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
        at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
        at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
        at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
        at java.net.Socket.connect(Socket.java:579)
        at java.net.Socket.connect(Socket.java:528)
        at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
        at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
        at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
        at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)
        at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
        at java.net.URL.openStream(URL.java:1037)
        at org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
	at __randomizedtesting.SeedInfo.seed([8BDBE10D84F0CD7E]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandlerBackup

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=18982, name=Thread-4347, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]         at java.net.PlainSocketImpl.socketConnect(Native Method)         at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)         at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)         at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)         at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)         at java.net.Socket.connect(Socket.java:579)         at java.net.Socket.connect(Socket.java:528)         at sun.net.NetworkClient.doConnect(NetworkClient.java:180)         at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)         at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)         at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)         at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)         at java.net.URL.openStream(URL.java:1037)         at org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=18982, name=Thread-4347, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]
        at java.net.PlainSocketImpl.socketConnect(Native Method)
        at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
        at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
        at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
        at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
        at java.net.Socket.connect(Socket.java:579)
        at java.net.Socket.connect(Socket.java:528)
        at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
        at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
        at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
        at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)
        at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
        at java.net.URL.openStream(URL.java:1037)
        at org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
	at __randomizedtesting.SeedInfo.seed([8BDBE10D84F0CD7E]:0)




Build Log:
[...truncated 11881 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReplicationHandlerBackup
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/init-core-data-001
   [junit4]   2> 1284846 T18962 oas.SolrTestCaseJ4.setUp ###Starting doTestBackup
   [junit4]   2> 1284871 T18962 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1284875 T18962 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:10045
   [junit4]   2> 1284875 T18962 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1284876 T18962 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1284876 T18962 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001
   [junit4]   2> 1284876 T18962 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/'
   [junit4]   2> 1284921 T18962 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/solr.xml
   [junit4]   2> 1285008 T18962 oasc.CoreContainer.<init> New CoreContainer 1515074261
   [junit4]   2> 1285008 T18962 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/]
   [junit4]   2> 1285010 T18962 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1285010 T18962 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1285010 T18962 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1285011 T18962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1285011 T18962 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1285011 T18962 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1285012 T18962 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1285012 T18962 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1285012 T18962 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1285013 T18962 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1285013 T18962 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1285014 T18962 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1285019 T18972 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/'
   [junit4]   2> 1285067 T18972 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 1285107 T18972 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1285108 T18972 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1285114 T18972 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1285141 T18972 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1285142 T18972 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir ./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/
   [junit4]   2> 1285142 T18972 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1285142 T18972 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/, dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/data/
   [junit4]   2> 1285142 T18972 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 1285143 T18972 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/data
   [junit4]   2> 1285143 T18972 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/data/index/
   [junit4]   2> 1285143 T18972 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1285144 T18972 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/data/index
   [junit4]   2> 1285145 T18972 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@43f4d098
   [junit4]   2> 1285145 T18972 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RateLimitedDirectoryWrapper(RAMDirectory@7550abed lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7daace02)),segFN=segments_1,generation=1}
   [junit4]   2> 1285146 T18972 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1285146 T18972 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1285147 T18972 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1285147 T18972 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1285147 T18972 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1285165 T18972 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1285167 T18972 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1285167 T18972 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1285168 T18972 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8391180582538591]
   [junit4]   2> 1285168 T18972 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RateLimitedDirectoryWrapper(RAMDirectory@7550abed lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7daace02)),segFN=segments_1,generation=1}
   [junit4]   2> 1285169 T18972 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1285169 T18972 oass.SolrIndexSearcher.<init> Opening Searcher@676d9ac6[collection1] main
   [junit4]   2> 1285169 T18972 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/conf
   [junit4]   2> 1285170 T18972 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/conf/}
   [junit4]   2> 1285170 T18972 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/conf
   [junit4]   2> 1285171 T18972 oasr.ManagedResource.reloadFromStorage WARN No stored data found for /rest/managed
   [junit4]   2> 1285171 T18972 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1285171 T18972 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1285172 T18972 oash.ReplicationHandler.inform Replication enabled for following config files: schema-replication2.xml:schema.xml
   [junit4]   2> 1285172 T18972 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1285172 T18973 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@676d9ac6[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1285172 T18972 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1285173 T18962 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0
   [junit4]   2> 1285173 T18962 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> ASYNC  NEW_CORE C908 name=collection1 org.apache.solr.core.SolrCore@57a7acf3
   [junit4]   2> 1285176 T18965 C908 oasu.DirectUpdateHandler2.deleteAll [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 1285177 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {deleteByQuery=*:*} 0 1
   [junit4]   2> 1285181 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[0]} 0 1
   [junit4]   2> 1285185 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[1]} 0 0
   [junit4]   2> 1285188 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[2]} 0 0
   [junit4]   2> 1285191 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[3]} 0 0
   [junit4]   2> 1285194 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[4]} 0 0
   [junit4]   2> 1285197 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[5]} 0 0
   [junit4]   2> 1285200 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[6]} 0 0
   [junit4]   2> 1285203 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[7]} 0 0
   [junit4]   2> 1285206 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[8]} 0 0
   [junit4]   2> 1285209 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[9]} 0 0
   [junit4]   2> 1285212 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[10]} 0 0
   [junit4]   2> 1285215 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[11]} 0 0
   [junit4]   2> 1285218 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[12]} 0 0
   [junit4]   2> 1285221 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[13]} 0 0
   [junit4]   2> 1285224 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[14]} 0 0
   [junit4]   2> 1285228 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[15]} 0 0
   [junit4]   2> 1285231 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[16]} 0 0
   [junit4]   2> 1285234 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[17]} 0 0
   [junit4]   2> 1285237 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[18]} 0 0
   [junit4]   2> 1285241 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[19]} 0 0
   [junit4]   2> 1285244 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[20]} 0 0
   [junit4]   2> 1285248 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[21]} 0 0
   [junit4]   2> 1285251 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[22]} 0 0
   [junit4]   2> 1285255 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[23]} 0 0
   [junit4]   2> 1285258 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[24]} 0 0
   [junit4]   2> 1285261 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[25]} 0 0
   [junit4]   2> 1285264 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[26]} 0 0
   [junit4]   2> 1285267 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[27]} 0 0
   [junit4]   2> 1285270 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[28]} 0 0
   [junit4]   2> 1285273 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[29]} 0 0
   [junit4]   2> 1285276 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[30]} 0 0
   [junit4]   2> 1285279 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[31]} 0 0
   [junit4]   2> 1285282 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[32]} 0 0
   [junit4]   2> 1285285 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[33]} 0 0
   [junit4]   2> 1285288 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[34]} 0 0
   [junit4]   2> 1285291 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[35]} 0 0
   [junit4]   2> 1285295 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[36]} 0 0
   [junit4]   2> 1285298 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[37]} 0 0
   [junit4]   2> 1285301 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[38]} 0 0
   [junit4]   2> 1285304 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[39]} 0 0
   [junit4]   2> 1285307 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[40]} 0 0
   [junit4]   2> 1285310 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[41]} 0 0
   [junit4]   2> 1285313 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[42]} 0 0
   [junit4]   2> 1285316 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[43]} 0 0
   [junit4]   2> 1285319 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[44]} 0 0
   [junit4]   2> 1285322 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[45]} 0 0
   [junit4]   2> 1285325 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[46]} 0 0
   [junit4]   2> 1285328 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[47]} 0 0
   [junit4]   2> 1285331 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[48]} 0 0
   [junit4]   2> 1285334 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[49]} 0 0
   [junit4]   2> 1285337 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[50]} 0 0
   [junit4]   2> 1285341 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[51]} 0 1
   [junit4]   2> 1285344 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[52]} 0 0
   [junit4]   2> 1285347 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[53]} 0 0
   [junit4]   2> 1285350 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[54]} 0 0
   [junit4]   2> 1285353 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[55]} 0 0
   [junit4]   2> 1285356 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[56]} 0 0
   [junit4]   2> 1285359 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[57]} 0 0
   [junit4]   2> 1285362 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[58]} 0 0
   [junit4]   2> 1285366 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[59]} 0 0
   [junit4]   2> 1285370 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[60]} 0 0
   [junit4]   2> 1285373 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[61]} 0 0
   [junit4]   2> 1285376 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[62]} 0 0
   [junit4]   2> 1285379 T18965 C908 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1285382 T18965 C908 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RateLimitedDirectoryWrapper(RAMDirectory@7550abed lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7daace02)),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RateLimitedDirectoryWrapper(RAMDirectory@7550abed lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7daace02)),segFN=segments_2,generation=2}
   [junit4]   2> 1285382 T18965 C908 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 1285384 T18965 C908 oass.SolrIndexSearcher.<init> Opening Searcher@14609e20[collection1] main
   [junit4]   2> 1285384 T18965 C908 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 1285384 T18973 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@14609e20[collection1] main{StandardDirectoryReader(segments_2:4:nrt _0(4.10):C63)}
   [junit4]   2> 1285385 T18965 C908 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={softCommit=false&waitSearcher=true&version=2&wt=javabin&commit=true} {commit=} 0 6
   [junit4]   2> 1285417 T18967 C908 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=details} status=0 QTime=2 
   [junit4]   2> 1285443 T18966 C908 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={name=pwnowpjztcfbxfzhbga&command=backup} status=0 QTime=28 
   [junit4]   2> 1285443 T18977 oash.SnapShooter.createSnapshot Creating backup snapshot...
   [junit4]   2> 1285467 T18977 oash.SnapShooter.createSnapshot Done creating backup snapshot, completed at: Fri Jul 04 19:18:08 LHST 2014
   [junit4]   2> 1285468 T18977 oash.SnapShooter.createSnapshot Done creating backup snapshot: pwnowpjztcfbxfzhbga
   [junit4]   2> 1285623 T18966 C908 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=details} status=0 QTime=0 
   [junit4]   2> 1286633 T18966 C908 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=details} status=0 QTime=0 
   [junit4]   2> 1286635 T18967 C908 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={name=eabfwnevyxleqy&command=backup} status=0 QTime=2 
   [junit4]   2> 1286635 T18979 oash.SnapShooter.createSnapshot Creating backup snapshot...
   [junit4]   2> 1286655 T18979 oash.SnapShooter.createSnapshot Done creating backup snapshot, completed at: Fri Jul 04 19:18:09 LHST 2014
   [junit4]   2> 1286656 T18979 oash.SnapShooter.createSnapshot Done creating backup snapshot: eabfwnevyxleqy
   [junit4]   2> 1286837 T18967 C908 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=details} status=0 QTime=0 
   [junit4]   2> 1286844 T18966 C908 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=details} status=0 QTime=0 
   [junit4]   2> 1286849 T18967 C908 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={name=pwnowpjztcfbxfzhbga&command=deletebackup} status=0 QTime=6 
   [junit4]   2> 1286849 T18981 oash.SnapShooter.deleteNamedSnapshot Deleting snapshot: pwnowpjztcfbxfzhbga
   [junit4]   2> 1287047 T18967 C908 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=details} status=0 QTime=0 
   [junit4]   2> 1287049 T18967 C908 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={command=details} status=0 QTime=0 
   [junit4]   2> 1287050 T18962 oas.SolrTestCaseJ4.tearDown ###Ending doTestBackup
   [junit4]   2> 1287050 T18983 oash.SnapShooter.deleteNamedSnapshot Deleting snapshot: eabfwnevyxleqy
   [junit4]   2> 1287050 T18962 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1515074261
   [junit4]   2> 1287050 T18966 C908 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={name=eabfwnevyxleqy&command=deletebackup} status=0 QTime=1 
   [junit4]   2> 1287052 T18962 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 1287053 T18966 C908 oasc.SolrException.log ERROR null:org.eclipse.jetty.io.EofException
   [junit4]   2> 		at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:142)
   [junit4]   2> 		at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:107)
   [junit4]   2> 		at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
   [junit4]   2> 		at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
   [junit4]   2> 		at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295)
   [junit4]   2> 		at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
   [junit4]   2> 		at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
   [junit4]   2> 		at org.apache.solr.util.FastWriter.flush(FastWriter.java:137)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.writeResponse(SolrDispatchFilter.java:766)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:426)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]   2> 		at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
   [junit4]   2> 		at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
   [junit4]   2> 		at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
   [junit4]   2> 		at org.eclipse.jetty.server.Server.handle(Server.java:368)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
   [junit4]   2> 		at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
   [junit4]   2> 		at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
   [junit4]   2> 		at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
   [junit4]   2> 		at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
   [junit4]   2> 		at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]   2> 		at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 		at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 		at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	
   [junit4]   2> 1287055 T18966 C908 oasc.SolrException.log ERROR null:org.eclipse.jetty.io.EofException
   [junit4]   2> 		at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:142)
   [junit4]   2> 		at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:107)
   [junit4]   2> 		at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
   [junit4]   2> 		at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
   [junit4]   2> 		at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295)
   [junit4]   2> 		at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
   [junit4]   2> 		at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
   [junit4]   2> 		at org.apache.solr.util.FastWriter.flush(FastWriter.java:137)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.writeResponse(SolrDispatchFilter.java:766)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:426)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]   2> 		at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
   [junit4]   2> 		at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
   [junit4]   2> 		at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
   [junit4]   2> 		at org.eclipse.jetty.server.Server.handle(Server.java:368)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
   [junit4]   2> 		at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
   [junit4]   2> 		at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
   [junit4]   2> 		at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
   [junit4]   2> 		at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
   [junit4]   2> 		at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]   2> 		at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 		at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 		at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	
   [junit4]   2> 1287056 T18966 C908 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@57a7acf3
   [junit4]   2> 1287056 T18966 C908 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=63,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0}
   [junit4]   2> 1287057 T18966 C908 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1287057 T18966 C908 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1287057 T18966 C908 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1287058 T18966 C908 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1287059 T18966 C908 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1287059 T18966 C908 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/data;done=false>>]
   [junit4]   2> 1287059 T18966 C908 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/data
   [junit4]   2> 1287060 T18966 C908 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/data/index;done=false>>]
   [junit4]   2> 1287060 T18966 C908 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solr.handler.TestReplicationHandlerBackup-8BDBE10D84F0CD7E-001/solr-instance-001/collection1/data/index
   [junit4]   2> 1287113 T18962 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 2275 T18961 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> 22385 T18961 ccr.ThreadLeakControl.checkThreadLeaks SEVERE 1 thread leaked from SUITE scope at org.apache.solr.handler.TestReplicationHandlerBackup: 
   [junit4]   2> 	   1) Thread[id=18982, name=Thread-4347, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]
   [junit4]   2> 	        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2> 	        at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
   [junit4]   2> 	        at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
   [junit4]   2> 	        at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
   [junit4]   2> 	        at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2> 	        at java.net.Socket.connect(Socket.java:579)
   [junit4]   2> 	        at java.net.Socket.connect(Socket.java:528)
   [junit4]   2> 	        at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
   [junit4]   2> 	        at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
   [junit4]   2> 	        at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
   [junit4]   2> 	        at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)
   [junit4]   2> 	        at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
   [junit4]   2> 	        at java.net.URL.openStream(URL.java:1037)
   [junit4]   2> 	        at org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
   [junit4]   2> 22386 T18961 ccr.ThreadLeakControl.tryToInterruptAll Starting to interrupt leaked threads:
   [junit4]   2> 	   1) Thread[id=18982, name=Thread-4347, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]
   [junit4]   2> 25392 T18961 ccr.ThreadLeakControl.tryToInterruptAll SEVERE There are still zombie threads that couldn't be terminated:
   [junit4]   2> 	   1) Thread[id=18982, name=Thread-4347, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]
   [junit4]   2> 	        at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]   2> 	        at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
   [junit4]   2> 	        at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
   [junit4]   2> 	        at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
   [junit4]   2> 	        at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]   2> 	        at java.net.Socket.connect(Socket.java:579)
   [junit4]   2> 	        at java.net.Socket.connect(Socket.java:528)
   [junit4]   2> 	        at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
   [junit4]   2> 	        at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
   [junit4]   2> 	        at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
   [junit4]   2> 	        at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)
   [junit4]   2> 	        at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
   [junit4]   2> 	        at java.net.URL.openStream(URL.java:1037)
   [junit4]   2> 	        at org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
   [junit4]   2> NOTE: test params are: codec=Lucene49: {name=PostingsFormat(name=MockRandom), id=PostingsFormat(name=MockRandom)}, docValues:{}, sim=DefaultSimilarity, locale=tr, timezone=Australia/LHI
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=2,free=140971088,total=498597888
   [junit4]   2> NOTE: All tests run in this JVM: [TestLMJelinekMercerSimilarityFactory, TestSerializedLuceneMatchVersion, TestSolrXMLSerializer, NotRequiredUniqueKeyTest, TestFieldResource, TestBinaryResponseWriter, TestRequestStatusCollectionAPI, DeleteShardTest, DistribDocExpirationUpdateProcessorTest, TestConfig, TestWordDelimiterFilterFactory, BadComponentTest, CoreAdminCreateDiscoverTest, PingRequestHandlerTest, XmlUpdateRequestHandlerTest, TestSolrDeletionPolicy1, MoreLikeThisHandlerTest, TestDefaultSimilarityFactory, TestInfoStreamLogging, EchoParamsTest, TestSolr4Spatial, TestFaceting, AnalyticsMergeStrategyTest, SpellPossibilityIteratorTest, SuggesterTSTTest, TestArbitraryIndexDir, TestJmxIntegration, PrimitiveFieldTypeTest, TestCustomSort, DistributedDebugComponentTest, TestStressRecovery, CustomCollectionTest, SuggesterTest, AtomicUpdatesTest, SuggestComponentTest, HighlighterConfigTest, TestSolrCoreProperties, SolrXmlInZkTest, TermsComponentTest, DirectSolrSpellCheckerTest, JSONWriterTest, TestDynamicFieldResource, TestAnalyzedSuggestions, RequiredFieldsTest, TestSchemaVersionResource, MinimalSchemaTest, CoreAdminHandlerTest, DateMathParserTest, DocValuesMissingTest, NoCacheHeaderTest, StressHdfsTest, TestQuerySenderListener, DirectUpdateHandlerTest, TestDistributedSearch, BasicFunctionalityTest, CircularListTest, ShardRoutingTest, OverseerCollectionProcessorTest, UpdateParamsTest, SliceStateTest, TestSolrIndexConfig, TestSort, PeerSyncTest, MergeStrategyTest, ExternalFileFieldSortTest, SSLMigrationTest, TestFoldingMultitermQuery, ZkControllerTest, HdfsCollectionsAPIDistributedZkTest, BasicDistributedZkTest, RecoveryZkTest, TestPartialUpdateDeduplication, TestAnalyzeInfixSuggestions, ResponseHeaderTest, TestLuceneMatchVersion, TestNumberUtils, PathHierarchyTokenizerFactoryTest, BadCopyFieldTest, TestTrie, BadIndexSchemaTest, TestManagedSynonymFilterFactory, SyncSliceTest, TestExpandComponent, DocumentAnalysisRequestHandlerTest, TestReplicationHandlerBackup]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestReplicationHandlerBackup -Dtests.seed=8BDBE10D84F0CD7E -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/hudson/lucene-data/enwiki.random.lines.txt -Dtests.locale=tr -Dtests.timezone=Australia/LHI -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | TestReplicationHandlerBackup (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.TestReplicationHandlerBackup: 
   [junit4]    >    1) Thread[id=18982, name=Thread-4347, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]
   [junit4]    >         at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]    >         at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
   [junit4]    >         at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
   [junit4]    >         at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
   [junit4]    >         at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]    >         at java.net.Socket.connect(Socket.java:579)
   [junit4]    >         at java.net.Socket.connect(Socket.java:528)
   [junit4]    >         at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
   [junit4]    >         at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
   [junit4]    >         at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
   [junit4]    >         at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)
   [junit4]    >         at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
   [junit4]    >         at java.net.URL.openStream(URL.java:1037)
   [junit4]    >         at org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8BDBE10D84F0CD7E]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=18982, name=Thread-4347, state=RUNNABLE, group=TGRP-TestReplicationHandlerBackup]
   [junit4]    >         at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]    >         at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
   [junit4]    >         at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
   [junit4]    >         at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
   [junit4]    >         at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]    >         at java.net.Socket.connect(Socket.java:579)
   [junit4]    >         at java.net.Socket.connect(Socket.java:528)
   [junit4]    >         at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
   [junit4]    >         at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
   [junit4]    >         at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
   [junit4]    >         at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:652)
   [junit4]    >         at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
   [junit4]    >         at java.net.URL.openStream(URL.java:1037)
   [junit4]    >         at org.apache.solr.handler.TestReplicationHandlerBackup$BackupThread.run(TestReplicationHandlerBackup.java:314)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8BDBE10D84F0CD7E]:0)
   [junit4] Completed on J0 in 25.45s, 1 test, 2 errors <<< FAILURES!

[...truncated 741 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:474: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/common-build.xml:496: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1299: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:923: There were test failures: 404 suites, 1654 tests, 2 suite-level errors, 845 ignored (7 assumptions)

Total time: 168 minutes 38 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-4.x #567
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 460295674 bytes
Compression is 0.0%
Took 1 min 11 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure